0

I want to build an camera Android app based on OpenCv in Kivy, then send the frame to the server with UDP. The server will use the frame to do something else,but it shoud be nparray. I'm inspired from here.

this is my code

server

import socket 
import cv2
import numpy as np

HOST = '127.0.0.1'
PORT = 5067

s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind((HOST, PORT))

print('server start at: %s:%s' % (HOST, PORT))
print('wait for connection...')

while True:
    indata, addr = s.recvfrom(1024)
    print('recvfrom ' + str(addr) + ': ' + indata.decode())
    #cv2.imshow('frame',frame)
    outdata = 'echo ACK!'
    s.sendto(outdata.encode(), addr)

server image

client

main.py

import string
from kivy.app import App
from kivy.uix.camera import Camera
from kivy.uix.boxlayout import BoxLayout
from kivy.lang import Builder
from kivy.clock import Clock
import numpy as np
import cv2
import socket

Builder.load_file('myapplayout.kv')


HOST = '127.0.0.1'
PORT = 5067
server_addr = (HOST, PORT)
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)


class AndroidCamera(Camera):
    camera_resolution = (640, 480)
    cam_ratio = camera_resolution[0] / camera_resolution[1]

class MyLayout(BoxLayout):
    pass

class MyApp(App):
    counter = 0
    zero = 0
    def build(self):
        return MyLayout()

    def on_start(self):
        Clock.schedule_once(self.get_frame, 5)

    def get_frame(self, dt):
        cam = self.root.ids.a_cam
        image_object = cam.export_as_image(scale=round((400 / int(cam.height)), 2))
        w, h = image_object._texture.size
        frame = np.frombuffer(image_object._texture.pixels, 'uint8').reshape(h, w, 4) #h:398 w:664
        #gray = cv2.cvtColor(frame, cv2.COLOR_RGBA2GRAY)
        outdata = str(frame)
        s.sendto(outdata.encode(), server_addr)

        indata, addr = s.recvfrom(1024)
        print('recvfrom ' + str(addr) + ': ' + indata.decode())

        cv2.imshow('frame',frame)
        try:
            self.root.ids.frame_counter.text = f'frame: {self.counter}'
        except:
            self.root.ids.frame_counter.text = f'frame: {self.zero}'
        self.counter += 1
        Clock.schedule_once(self.get_frame, 0.25)

if __name__ == "__main__":
    MyApp().run()

client image

myapplayout.kv

<MyLayout>:
    orientation: 'vertical'
    size: root.width, root.height

    GridLayout:
        rows: 2

        RelativeLayout:
            size_hint: 1, 0.8

            AndroidCamera:
                index: 0
                id: a_cam
                resolution: self.camera_resolution
                allow_stretch: True
                play: True
                canvas.before:
                    PushMatrix
                    Rotate:
                        angle: -90
                        origin: self.center
                    Scale:
                        x: self.cam_ratio
                        y: self.cam_ratio
                        origin: self.center
                canvas.after:
                    PopMatrix

        Label:
            size_hint: 1, 0.2
            id: frame_counter
            font_size: self.height * 0.4
            text: ''

buildozer.spec (I follow here and above link)

[app]

# (str) Title of your application
title = opencv demo

# (str) Package name
package.name = opencvdemo

# (str) Package domain (needed for android/ios packaging)
package.domain = cool.lr

# (str) Source code where the main.py live
source.dir = .

# (list) Source files to include (let empty to include all the files)
#source.include_exts = py,png,jpg,kv,atlas

# (list) List of inclusions using pattern matching
#source.include_patterns = assets/*,images/*.png

# (list) Source files to exclude (let empty to not exclude anything)
#source.exclude_exts = spec

# (list) List of directory to exclude (let empty to not exclude anything)
#source.exclude_dirs = tests, bin

# (list) List of exclusions using pattern matching
#source.exclude_patterns = license,images/*/*.jpg

# (str) Application versioning (method 1)
version = 0.1

# (str) Application versioning (method 2)
# version.regex = __version__ = ['"](.*)['"]
# version.filename = %(source.dir)s/main.py

# (list) Application requirements
# comma seperated e.g. requirements = sqlite3,kivy
requirements = kivy,python3,numpy,opencv

# (str) Custom source folders for requirements
# Sets custom source for any requirements with recipes
# requirements.source.kivy = ../../kivy

# (list) Garden requirements
#garden_requirements =

# (str) Presplash of the application
#presplash.filename = %(source.dir)s/data/presplash.png

# (str) Icon of the application
#icon.filename = %(source.dir)s/data/icon.png

# (str) Supported orientation (one of landscape, portrait or all)
orientation = portrait

# (list) List of service to declare
#services = NAME:ENTRYPOINT_TO_PY,NAME2:ENTRYPOINT2_TO_PY

#
# OSX Specific
#

#
# author = © Copyright Info

#
# Android specific
#

# (bool) Indicate if the application should be fullscreen or not
fullscreen = 0

# (list) Permissions
android.permissions = CAMERA

# (int) Android API to use
#android.api = 19

# (int) Minimum API required
#android.minapi = 9

# (int) Android SDK version to use
#android.sdk = 20

# (str) Android NDK version to use
#android.ndk = 9c

# (bool) Use --private data storage (True) or --dir public storage (False)
#android.private_storage = True

# (str) Android NDK directory (if empty, it will be automatically downloaded.)
#android.ndk_path =

# (str) Android SDK directory (if empty, it will be automatically downloaded.)
#android.sdk_path =

# (str) ANT directory (if empty, it will be automatically downloaded.)
#android.ant_path =

# (str) python-for-android git clone directory (if empty, it will be automatically cloned from github)
#android.p4a_dir =

# (list) python-for-android whitelist
#android.p4a_whitelist =

# (bool) If True, then skip trying to update the Android sdk
# This can be useful to avoid excess Internet downloads or save time
# when an update is due and you just want to test/build your package
# android.skip_update = False

# (str) Android entry point, default is ok for Kivy-based app
#android.entrypoint = org.renpy.android.PythonActivity

# (list) List of Java .jar files to add to the libs so that pyjnius can access
# their classes. Don't add jars that you do not need, since extra jars can slow
# down the build process. Allows wildcards matching, for example:
# OUYA-ODK/libs/*.jar
#android.add_jars = foo.jar,bar.jar,path/to/more/*.jar

# (list) List of Java files to add to the android project (can be java or a
# directory containing the files)
#android.add_src =

# (str) python-for-android branch to use, if not master, useful to try
# not yet merged features.
#android.branch = master

# (str) OUYA Console category. Should be one of GAME or APP
# If you leave this blank, OUYA support will not be enabled
#android.ouya.category = GAME

# (str) Filename of OUYA Console icon. It must be a 732x412 png image.
#android.ouya.icon.filename = %(source.dir)s/data/ouya_icon.png

# (str) XML file to include as an intent filters in <activity> tag
#android.manifest.intent_filters =

# (list) Android additionnal libraries to copy into libs/armeabi
#android.add_libs_armeabi = libs/android/*.so
#android.add_libs_armeabi_v7a = libs/android-v7/*.so
#android.add_libs_x86 = libs/android-x86/*.so
#android.add_libs_mips = libs/android-mips/*.so

# (bool) Indicate whether the screen should stay on
# Don't forget to add the WAKE_LOCK permission if you set this to True
#android.wakelock = False

# (list) Android application meta-data to set (key=value format)
#android.meta_data =

# (list) Android library project to add (will be added in the
# project.properties automatically.)
#android.library_references =

# (str) Android logcat filters to use
#android.logcat_filters = *:S python:D

# (bool) Copy library instead of making a libpymodules.so
#android.copy_libs = 1

#
# iOS specific
#

# (str) Path to a custom kivy-ios folder
#ios.kivy_ios_dir = ../kivy-ios

# (str) Name of the certificate to use for signing the debug version
# Get a list of available identities: buildozer ios list_identities
#ios.codesign.debug = "iPhone Developer: <lastname> <firstname> (<hexstring>)"

# (str) Name of the certificate to use for signing the release version
#ios.codesign.release = %(ios.codesign.debug)s


[buildozer]

# (int) Log level (0 = error only, 1 = info, 2 = debug (with command output))
log_level = 2

# (int) Display warning if buildozer is run as root (0 = False, 1 = True)
warn_on_root = 0

# (str) Path to build artifact storage, absolute or relative to spec file
# build_dir = ./.buildozer

# (str) Path to build output (i.e. .apk, .ipa) storage
# bin_dir = ./bin

#    -----------------------------------------------------------------------------
#    List as sections
#
#    You can define all the "list" as [section:key].
#    Each line will be considered as a option to the list.
#    Let's take [app] / source.exclude_patterns.
#    Instead of doing:
#
#[app]
#source.exclude_patterns = license,data/audio/*.wav,data/images/original/*
#
#    This can be translated into:
#
#[app:source.exclude_patterns]
#license
#data/audio/*.wav
#data/images/original/*
#


#    -----------------------------------------------------------------------------
#    Profiles
#
#    You can extend section / key with a profile
#    For example, you want to deploy a demo version of your application without
#    HD content. You could first change the title to add "(demo)" in the name
#    and extend the excluded directories to remove the HD content.
#
#[app@demo]
#title = My Application (demo)
#
#[app:source.exclude_patterns@demo]
#images/hd/*
#
#    Then, invoke the command line with the "demo" profile:
#
#buildozer --profile demo android debug

I need to do some cv2 operators at server, but I don't know how to change indata.decode() (this is str) to original nparray just like the frame in client's main.py.

Markus
  • 5,976
  • 5
  • 6
  • 21
rayin
  • 1

1 Answers1

0

You can use tobytes to get the image raw data from a numpy array and send it to the server like this:

img = np.array([[0, 1], [2, 3]], dtype='uint8')  # 2x2 example
s.sendto(img.tobytes())

On server side get the image data from the buffer and reshape it to the original shape:

buffer, addr = s.recv(2*2)  # receive data
np.frombuffer(buffer, dtype='uint8').reshape(2,2)

Keep in mind that you always should know how much bytes you will receive. For this it is usual to send information about the size as a header. Example in pseudo code:

Client:

  • convert shape of image as uint32-array to bytes
  • convert image data as uint8-array to bytes
  • send size of shape byte data as uint32
  • send size of image byte data as uint32
  • send shape byte data
  • send image byte data

Server:

  • recv first 4 bytes and convert them to uint32 as size of shape data (shape_size)
  • recv next 4 bytes and convert them to uint32 as size of image data (img_size)
  • recv next shape_size bytes and convert them to uint32-numpy-array
  • recv next img_size bytes and convert them to uint8-numpy-array
  • resize image data by shape data
Markus
  • 5,976
  • 5
  • 6
  • 21