commit by robo pro coding

.project.json

+{"uuid":"e2679fc1-bbd2-5142-2ec8-ad7e1cab3590","name":"Add_On_AI","mode":"ADVANCED","version":"1.0","controller":"TXT4"}
\ No newline at end of file

Add_On_AI.blockly

+<xml xmlns="https://developers.google.com/blockly/xml" version="14">
+  <block type="start_block" id="$#]ESP+D}J$B]GY;Y`zp" deletable="false" x="0" y="125">
+    <statement name="statement">
+      <block type="util_thread" id="nOZkJ)D$n=;X^0bV*uy_">
+        <field name="dropdown_event">]$}!d~iPi$faCfYn-,02</field>
+        <next>
+          <block type="loop_endless" id="]bji@o6zCyG=6j.U__2I"/>
+        </next>
+      </block>
+    </statement>
+  </block>
+</xml>
\ No newline at end of file

Add_On_AI.py

+import threading
+
+from lib.controller import *
+from lib.machine_learning import *
+from lib.sorting_line import *
+
+
+threading.Thread(target=thread_SLD, daemon=True).start()
+while True:
+    pass

lib/camera.py

+# auto generated content from camera configuration
+from lib.controller import *
+import fischertechnik.factories as txt_factory
+
+TXT_SLD_M_USB1_1_camera.set_rotate(False)
+TXT_SLD_M_USB1_1_camera.set_height(240)
+TXT_SLD_M_USB1_1_camera.set_width(320)
+TXT_SLD_M_USB1_1_camera.set_fps(15)
+TXT_SLD_M_USB1_1_camera.start()
+

lib/camera.xml

+<xml type="camera" version="2" camera_name="TXT_SLD_M_USB1_1_camera"/>
\ No newline at end of file

lib/controller.blockly

+<xml xmlns="https://developers.google.com/blockly/xml" version="14">
+  <block type="txt4_controller" id="[v6x3J=Tm+dMJAsQlQOi" x="588" y="-472">
+    <mutation>
+      <arg type="Input" name="Input-0" value="I1"/>
+      <arg type="Input" name="Input-1" value="I2"/>
+      <arg type="Input" name="Input-2" value="-"/>
+      <arg type="Output" name="Output-0" value="O5"/>
+      <arg type="Output" name="Output-1" value="-"/>
+      <arg type="Motor" name="Motor-0" value="M1"/>
+      <arg type="Motor" name="Motor-1" value="M2"/>
+      <arg type="Motor" name="Motor-2" value="-"/>
+      <arg type="Servomotor" name="Servomotor-0" value="-"/>
+      <arg type="Counter" name="Counter-0" value="C1"/>
+      <arg type="Counter" name="Counter-1" value="C2"/>
+      <arg type="Counter" name="Counter-2" value="-"/>
+      <arg type="I2C" name="I2C-0" value="-"/>
+      <arg type="USB" name="USB-0" value="USB1_1"/>
+      <arg type="USB" name="USB-1" value="-"/>
+    </mutation>
+    <field name="mode">0</field>
+    <field name="controller_name">TXT_SLD</field>
+    <field name="Input-0">I1</field>
+    <field name="Input-1">I2</field>
+    <field name="Input-2">-</field>
+    <field name="Output-0">O5</field>
+    <field name="Output-1">-</field>
+    <field name="Motor-0">M1</field>
+    <field name="Motor-1">M2</field>
+    <field name="Motor-2">-</field>
+    <field name="Servomotor-0">-</field>
+    <field name="Counter-0">C1</field>
+    <field name="Counter-1">C2</field>
+    <field name="Counter-2">-</field>
+    <field name="I2C-0">-</field>
+    <field name="USB-0">USB1_1</field>
+    <field name="USB-1">-</field>
+    <value name="Input-0">
+      <block type="photo_transistor" id="q8b-=jiA!QN*/0L=SU|;"/>
+    </value>
+    <value name="Input-1">
+      <block type="mini_switch" id="0/,xi7n3}c2!E^qfOh?$"/>
+    </value>
+    <value name="Output-0">
+      <block type="led" id=":^baCXt_)quR(#FAUAv_"/>
+    </value>
+    <value name="Motor-0">
+      <block type="encodermotor" id="ueH%w!w?!F?z}pP!QiP7"/>
+    </value>
+    <value name="Motor-1">
+      <block type="encodermotor" id=")Eq*8C-!QYXfX/2!QXeH"/>
+    </value>
+    <value name="Counter-0">
+      <block type="motor_step_counter" id="+}L[:_O.2.zb0y*J51c0">
+        <field name="instance_name">M1</field>
+      </block>
+    </value>
+    <value name="Counter-1">
+      <block type="motor_step_counter" id="~A-oSC6qEM5/jp#5FwaR">
+        <field name="instance_name">M2</field>
+      </block>
+    </value>
+    <value name="USB-0">
+      <block type="camera" id=".[.Yo}=Loo#hh0Ws+A]i"/>
+    </value>
+  </block>
+</xml>
\ No newline at end of file

lib/controller.py

+import fischertechnik.factories as txt_factory
+
+txt_factory.init()
+txt_factory.init_input_factory()
+txt_factory.init_output_factory()
+txt_factory.init_motor_factory()
+txt_factory.init_counter_factory()
+txt_factory.init_usb_factory()
+txt_factory.init_camera_factory()
+
+TXT_SLD_M = txt_factory.controller_factory.create_graphical_controller()
+TXT_SLD_M_I1_photo_transistor = txt_factory.input_factory.create_photo_transistor(TXT_SLD_M, 1)
+TXT_SLD_M_I2_mini_switch = txt_factory.input_factory.create_mini_switch(TXT_SLD_M, 2)
+TXT_SLD_M_O5_led = txt_factory.output_factory.create_led(TXT_SLD_M, 5)
+TXT_SLD_M_M1_encodermotor = txt_factory.motor_factory.create_encodermotor(TXT_SLD_M, 1)
+TXT_SLD_M_M2_encodermotor = txt_factory.motor_factory.create_encodermotor(TXT_SLD_M, 2)
+TXT_SLD_M_C1_motor_step_counter = txt_factory.counter_factory.create_encodermotor_counter(TXT_SLD_M, 1)
+TXT_SLD_M_C1_motor_step_counter.set_motor(TXT_SLD_M_M1_encodermotor)
+TXT_SLD_M_C2_motor_step_counter = txt_factory.counter_factory.create_encodermotor_counter(TXT_SLD_M, 2)
+TXT_SLD_M_C2_motor_step_counter.set_motor(TXT_SLD_M_M2_encodermotor)
+TXT_SLD_M_USB1_1_camera = txt_factory.usb_factory.create_camera(TXT_SLD_M, 1)
+
+txt_factory.initialized()
\ No newline at end of file

lib/display.py

+import os
+import time
+import threading
+import signal
+
+import ftgui
+
+display = ftgui.fttxt2_gui_connector("app")
+display.open()
+
+def display_monitoring():
+    while display.is_open():
+        time.sleep(1)
+    os.kill(os.getpid(), signal.SIGTERM)
+    exit()
+
+threading.Thread(target=display_monitoring, daemon=True).start()

lib/display.qml

+// auto generated content from display configuration
+import QtQuick 2.2
+import QtQuick.Window 2.0
+import QtQuick.Controls 1.1
+import QtQuick.Controls.Styles 1.1
+import QtQuick.Extras 1.4
+
+TXTWindow {
+  Rectangle {
+    id: rect
+    color: "grey"
+    anchors.fill: parent
+  }
+  TXTLabel {
+    id: img_label
+    text: ""
+    font.pixelSize: 16
+    elide: Text.ElideRight
+    x: 20
+    y: 20
+    width: 200
+    height: 150
+  }
+  StatusIndicator {
+    id: red
+    color: "#EC1600"
+    active: false
+    x: 20
+    y: 195
+    width: 35
+    height: 35
+  }
+  StatusIndicator {
+    id: blue
+    color: "#005693"
+    active: false
+    x: 100
+    y: 195
+    width: 35
+    height: 35
+  }
+  StatusIndicator {
+    id: fail
+    color: "#E7AE13"
+    active: false
+    x: 185
+    y: 195
+    width: 35
+    height: 35
+  }
+  TXTLabel {
+    id: version_label
+    text: "Add-On AI: Version 2023/04/20"
+    font.pixelSize: 16
+    elide: Text.ElideRight
+    x: 20
+    y: 0
+    width: 200
+    height: 20
+  }
+  TXTLabel {
+    id: part_pass_fail
+    text: ""
+    font.pixelSize: 16
+    elide: Text.ElideRight
+    x: 20
+    y: 170
+    width: 200
+    height: 20
+  }
+}

lib/display.xml

+<xml type="display" version="2"><item id="33" class="TXTLabel"><name>img_label</name><text/><geometry><x>20</x><y>20</y><width>200</width><height>150</height></geometry></item><item id="20" class="StatusIndicator"><name>red</name><color>#EC1600</color><active>false</active><geometry><x>20</x><y>195</y><width>35</width><height>35</height></geometry></item><item id="22" class="StatusIndicator"><name>blue</name><color>#005693</color><active>false</active><geometry><x>100</x><y>195</y><width>35</width><height>35</height></geometry></item><item id="23" class="StatusIndicator"><name>fail</name><color>#E7AE13</color><active>false</active><geometry><x>185</x><y>195</y><width>35</width><height>35</height></geometry></item><item id="24" class="TXTLabel"><name>version_label</name><text>Add-On AI: Version 2023/04/20</text><geometry><x>20</x><y>0</y><width>200</width><height>20</height></geometry></item><item id="28" class="TXTLabel"><name>part_pass_fail</name><text/><geometry><x>20</x><y>170</y><width>200</width><height>20</height></geometry></item></xml>
\ No newline at end of file

lib/machine_learning.py

+import base64
+import cv2
+import datetime
+import numpy as np
+import subprocess
+import time
+
+from datetime import datetime
+from fischertechnik.camera.VideoStream import VideoStream
+from fischertechnik.machine_learning.ObjectDetector import ObjectDetector
+from lib.camera import *
+from lib.controller import *
+from lib.display import *
+
+
+tag = None
+value = None
+num = None
+color = None
+ts = None
+filename = None
+sat = None
+hue = None
+duration = None
+prob = None
+keytext = None
+pos = None
+frame = None
+ts_process0 = None
+detector = None
+ts_process1 = None
+result = None
+ts_process = None
+key = None
+
+
+def MakePictureRunKiReturnFoundPart():
+    global tag, value, num, color, ts, filename, sat, hue, duration, prob, keytext, pos, frame, ts_process0, detector, ts_process1, result, ts_process, key
+    reset_inteface()
+    TXT_SLD_M_O5_led.set_brightness(int(512))
+    time.sleep(0.2)
+    TXT_SLD_M_O5_led.set_brightness(int(30))
+    time.sleep(0.8)
+    duration = (time.time() * 1000)
+    num = 4
+    prob = 0
+    keytext = 'No feature found'
+    pos = ''
+    display.set_attr("part_pass_fail.text", str(containInHTML('i', 'processing')))
+    frame = TXT_SLD_M_USB1_1_camera.read_frame()
+    #get color from frame
+    color = (np.mean(frame[ 80:120,  100:240], axis=(0, 1)))
+    color = cv2.cvtColor(np.uint8([[[color[0],color[1],color[2]]]]),cv2.COLOR_BGR2HLS)[0][0]
+    hue = color[0] # range 0-180
+    sat = color[2] # range 0-255
+    ts_process0 = (time.time() * 1000)
+    detector = ObjectDetector('/opt/ft/workspaces/machine-learning/object-detection/sorting_line/model.tflite', '/opt/ft/workspaces/machine-learning/object-detection/sorting_line/labels.txt')
+    ts_process1 = (time.time() * 1000)
+    result = detector.process_image(frame)
+    ts_process = (time.time() * 1000)
+    print('processing time: {:.0f} ms {:.0f} ms'.format(ts_process1 - ts_process0, ts_process - ts_process1))
+    color = get_color()
+    TXT_SLD_M_O5_led.set_brightness(int(0))
+    print(result)
+    if len(result) > 0:
+        prob = result[0]['probability']
+        pos = result[0]['position']
+        key = result[0]['label']
+        keytext = ''
+        if key == 'CRACK':
+            keytext = 'Cracks in Workpiece'
+        elif key == 'MIPO1':
+            keytext = '1x milled pocket'
+        elif key == 'MIPO2':
+            keytext = '2x milled pocket'
+        elif key == 'BOHO':
+            keytext = 'Round hole'
+        elif key == 'BOHOEL':
+            keytext = 'Hole elyptical'
+        elif key == 'BOHOMIPO1':
+            keytext = 'Hole and 1x milled pocket'
+        elif key == 'BOHOMIPO2':
+            keytext = 'Hole and 2x milled pocket'
+        elif key == 'BLANK':
+            keytext = 'Workpiece without features'
+        else:
+            keytext = key
+        print('{} {} {} {} {} {}'.format(key, keytext, num, prob, pos, color))
+        if key == 'BOHO' and color == 1:
+            num = 1
+            display.set_attr("red.active", str(True).lower())
+            display.set_attr("part_pass_fail.text", str(containInHTML('b', "Workpiece <font color='#88ff88'> PASSED</font>")))
+        elif key == 'MIPO2' and color == 2:
+            num = 2
+            display.set_attr("red.active", str(True).lower())
+            display.set_attr("part_pass_fail.text", str(containInHTML('b', "Workpiece <font color='#88ff88'> PASSED</font>")))
+        elif key == 'BOHOMIPO2' and color == 3:
+            num = 3
+            display.set_attr("blue.active", str(True).lower())
+            display.set_attr("part_pass_fail.text", str(containInHTML('b', "Workpiece <font color='#88ff88'> PASSED</font>")))
+        else:
+            num = 4
+            display.set_attr("fail.active", str(True).lower())
+            display.set_attr("part_pass_fail.text", str(containInHTML('b', "Workpiece <font color='#ff8888'>FAILED</font>")))
+    else:
+        display.set_attr("part_pass_fail.text", str(containInHTML('b', "Workpiece <font color='#ff8888'>FAILED</font>")))
+    duration = (time.time() * 1000) - duration
+    saveFileandPublish()
+    return num
+
+
+def reset_inteface():
+    global tag, value, num, color, ts, filename, sat, hue, duration, prob, keytext, pos, frame, ts_process0, detector, ts_process1, result, ts_process, key
+    display.set_attr("part_pass_fail.text", str(containInHTML('i', 'Not analysed yet')))
+    display.set_attr("red.active", str(False).lower())
+    display.set_attr("blue.active", str(False).lower())
+    display.set_attr("fail.active", str(False).lower())
+
+
+def containInHTML(tag, value):
+    global num, color, ts, filename, sat, hue, duration, prob, keytext, pos, frame, ts_process0, detector, ts_process1, result, ts_process, key
+    return ''.join([str(x) for x in ['<', tag, '>', value, '</', tag, '>']])
+
+
+def get_color():
+    global tag, value, num, color, ts, filename, sat, hue, duration, prob, keytext, pos, frame, ts_process0, detector, ts_process1, result, ts_process, key
+    if hue >= 85 and hue < 130 and sat >= 40:
+        color = 3
+    elif (hue >= 130 and hue <= 180 or hue >= 0 and hue < 15) and sat >= 40:
+        color = 2
+    else:
+        color = 1
+    return color
+
+
+def timestamp():
+    global tag, value, num, color, ts, filename, sat, hue, duration, prob, keytext, pos, frame, ts_process0, detector, ts_process1, result, ts_process, key
+    ts = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+    return ts
+
+
+def saveFileandPublish():
+    global tag, value, num, color, ts, filename, sat, hue, duration, prob, keytext, pos, frame, ts_process0, detector, ts_process1, result, ts_process, key
+    filename = '/opt/ft/workspaces/last-image.png'
+    if(pos != ""):
+        image = cv2.rectangle(frame, (pos[0], pos[1]), (pos[2], pos[3]), (180,105,0), 2)
+    print("write png file: ", filename)
+    cv2.imwrite(filename, frame)
+    subprocess.Popen(['chmod', '777', filename])
+
+    with open(filename, "rb") as img_file:
+        my_string = base64.b64encode(img_file.read())
+    imgb64 = "data:image/jpeg;base64," + (my_string.decode('utf-8'))
+    time.sleep(0.2)
+    displaystr= "<img width='200' height='150' src='" +  imgb64  + "'>"
+    display.set_attr("img_label.text", str(displaystr))
+
+

lib/sorting_line.py

+import time
+
+from fischertechnik.controller.Motor import Motor
+from lib.controller import *
+from lib.machine_learning import *
+
+
+MovementSpeed = None
+posRED = None
+posBLUE = None
+posCam = None
+i = None
+posFinish = None
+num = None
+
+
+def thread_SLD():
+    global MovementSpeed, posRED, posBLUE, posCam, i, posFinish, num
+    print('starting thread SLD')
+    MovementSpeed = 300
+    posRED = 50
+    posBLUE = 150
+    posCam = 40
+    posFinish = 400
+    num = -1
+    moveRefM2()
+    while True:
+        mainSLDexternal()
+
+
+def moveRefM2():
+    global MovementSpeed, posRED, posBLUE, posCam, i, posFinish, num
+    TXT_SLD_M_M2_encodermotor.set_speed(int(300), Motor.CCW)
+    TXT_SLD_M_M2_encodermotor.start()
+    while True:
+        if (TXT_SLD_M_I2_mini_switch.is_closed()):
+            break
+        time.sleep(0.010)
+    TXT_SLD_M_M2_encodermotor.stop()
+
+
+def upRange(start, stop, step):
+    while start <= stop:
+        yield start
+        start += abs(step)
+
+
+def downRange(start, stop, step):
+    while start >= stop:
+        yield start
+        start -= abs(step)
+
+
+def mainSLDexternal():
+    global MovementSpeed, posRED, posBLUE, posCam, i, posFinish, num
+    if TXT_SLD_M_I1_photo_transistor.is_dark():
+        reset_inteface()
+        print('start')
+        TXT_SLD_M_M1_encodermotor.set_speed(int(MovementSpeed * 0.5), Motor.CCW)
+        TXT_SLD_M_M1_encodermotor.start_sync()
+        for i in (0 <= float(posCam)) and upRange(0, float(posCam), 1) or downRange(0, float(posCam), 1):
+            print(i)
+            if not (TXT_SLD_M_I1_photo_transistor.is_dark()):
+                break
+            time.sleep(0.01)
+        TXT_SLD_M_M1_encodermotor.stop_sync()
+        TXT_SLD_M_M1_encodermotor.set_speed(int(MovementSpeed), Motor.CCW)
+        TXT_SLD_M_M1_encodermotor.set_distance(int(posCam))
+        while True:
+            if (not TXT_SLD_M_M1_encodermotor.is_running()):
+                break
+            time.sleep(0.010)
+        print('pos camera')
+        num = MakePictureRunKiReturnFoundPart()
+        print(num)
+        if num == 1:
+            TXT_SLD_M_M1_encodermotor.set_speed(int(MovementSpeed), Motor.CCW)
+            TXT_SLD_M_M1_encodermotor.set_distance(int(600))
+            while True:
+                if (not TXT_SLD_M_M1_encodermotor.is_running()):
+                    break
+                time.sleep(0.010)
+        elif num == 2:
+            print('RED')
+            TXT_SLD_M_M2_encodermotor.set_speed(int(300), Motor.CW)
+            TXT_SLD_M_M2_encodermotor.set_distance(int(posRED))
+            while True:
+                if (not TXT_SLD_M_M2_encodermotor.is_running()):
+                    break
+                time.sleep(0.010)
+            TXT_SLD_M_M1_encodermotor.set_speed(int(MovementSpeed), Motor.CCW)
+            TXT_SLD_M_M1_encodermotor.set_distance(int(posFinish))
+            while True:
+                if (not TXT_SLD_M_M1_encodermotor.is_running()):
+                    break
+                time.sleep(0.010)
+        elif num == 3:
+            print('BLUE')
+            TXT_SLD_M_M2_encodermotor.set_speed(int(300), Motor.CW)
+            TXT_SLD_M_M2_encodermotor.set_distance(int(posBLUE))
+            while True:
+                if (not TXT_SLD_M_M2_encodermotor.is_running()):
+                    break
+                time.sleep(0.010)
+            TXT_SLD_M_M1_encodermotor.set_speed(int(MovementSpeed), Motor.CCW)
+            TXT_SLD_M_M1_encodermotor.set_distance(int(posFinish))
+            while True:
+                if (not TXT_SLD_M_M1_encodermotor.is_running()):
+                    break
+                time.sleep(0.010)
+        else:
+            print('FAIL')
+            moveRefM2()
+            TXT_SLD_M_M1_encodermotor.set_speed(int(MovementSpeed), Motor.CCW)
+            TXT_SLD_M_M1_encodermotor.set_distance(int(posFinish))
+            while True:
+                if (not TXT_SLD_M_M1_encodermotor.is_running()):
+                    break
+                time.sleep(0.010)