Hi all, stay tuned for this topic...

I've made some progress since the 3.3 beta delivery and some new personal investigations watching ARKit headers.

... I'm refactoring my basic code sample...

@dgelessus : you're right ! I've extracted objc_util.py from my current Pythonista and have copied it to the XCode Template project (PythonistaAppTemplate/PythonistaKit.framework/pylib/site-packages directory ). Now, I can test my app :o)

@JonB : I don't need to add the framework I want to load in my xcode project. I've removed the framework in the xcode project and my app can be launched without any problem...Perhaps as it's an Apple framework, I don't need to add it....My xcode knowledge is a little low ;o)

Thank you for your help...

Edit: I found a solution, see here https://forum.omz-software.com/topic/4362/first-attempt-to-integrate-arkit-and-first-questions/29

Hi,

Following some objective-c/swift ARkit examples, I made some python tests using the objc_util module on my iPhone 6S+ with iOS 11 GM.
I move forward step by step... as I'm a complete beginner in "calling objective-c from python"...
My results:
I can open the camera through ARkit but I'm thinking my arscnview is not correctly initialized and my 'delegate' is not correctly set. For example, despite my call to setDebugOptions_, I can't display the feature points for example, or if I insert a beep in the renderer_didAdd_for_ method, I can't ear any sound even if a plane is supposed to be detected by my camera.
So if you have any advice, I'm more than interrested !o)

Here my code:

# coding: utf-8


from objc_util import *
import ui
import os
import sys
#from myDebugToolKit import *

load_framework('SceneKit')
load_framework('ARKit')

# Some 'constants' used by ARkit
# But i can't transfer them to the ARKit framework, why ?????
ARWorldAlignmentGravity = 0
ARWorldAlignmentGravityAndHeading = 1
ARWorldAlignmentCamera = 2
    
ARPlaneDetectionNone = 0
ARPlaneDetectionHorizontal = 1 << 0
ARPlaneDetectionVertical = 1 << 1

ARSCNDebugOptionNone = 0 
ARSCNDebugOptionShowWorldOrigin = 1 << 0
ARSCNDebugOptionShowFeaturePoints = 1 << 1
    



SCNScene = ObjCClass('SCNScene')
ARSCNView = ObjCClass('ARSCNView')
ARWorldTrackingConfiguration = ObjCClass('ARWorldTrackingConfiguration')
ARSession = ObjCClass('ARSession')
UIViewController = ObjCClass('UIViewController')
ARPlaneAnchor = ObjCClass('ARPlaneAnchor')



# I should refactor te following line in a class but I need to learn more the create_objcc_class function 
sceneview = None   

# Here two little set up functions used by the main class
def createSampleScene():
    # an empty scene
    scene = SCNScene.scene()
    return scene


def createARSceneView(x, y, w, h, debug=True):
    v = ARSCNView.alloc().initWithFrame_((CGRect(CGPoint(x, y), CGSize(w, h))))
    v.setShowsStatistics_(debug)
    # Problem here... feature points are not shown.... despite the method call
    v.setDebugOptions_(ARSCNDebugOptionShowWorldOrigin | ARSCNDebugOptionShowFeaturePoints)  
    return v

# Some callback definitions used by create_objc_class
def CustomViewController_touchesBegan_withEvent_(_self, _cmd, _touches, event):
    touches = ObjCInstance(_touches)
    for t in touches:
        loc = t.locationInView_(sceneview)
        sz = ui.get_screen_size()
        print(loc)

def CustomViewController_viewWillAppear_(_self, _cmd, animated):    
    configuration = ARWorldTrackingConfiguration.alloc().init()
    # Another problem here...constants aren't well communicated... (my assumption...)  
    configuration.setPlaneDetection_ (ARPlaneDetectionHorizontal)
    configuration.setWorldAlignment_(ARWorldAlignmentGravityAndHeading)
    
    session = sceneview.session()
    session.runWithConfiguration_(configuration)


def CustomViewController_viewWillDisappear_(_self, _cmd, animated):
    session = sceneview.session()   
    session.pause()
    

def MyARSCNViewDelegate_renderer_didAdd_for_(_self, _cmd, scenerenderer, node, anchor):
    if not isinstance(anchor, (ARPlaneAnchor)):
        return
    # to be implemented...

# The main class...
class MyARView(ui.View):
    def __init__(self):
        global sceneview
        self.flex = 'WH'
        
        screen = ui.get_screen_size()

        # set up the scene
        scene = createSampleScene()

        # set up the ar scene view delegate
        methods = [MyARSCNViewDelegate_renderer_didAdd_for_]
        protocols = ['ARSCNViewDelegate']
        MyARSCNViewDelegate = create_objc_class('MyARSCNViewDelegate', NSObject, methods=methods, protocols=protocols)
        delegate = MyARSCNViewDelegate.alloc().init()

        # set up the ar scene view
        sceneview = createARSceneView(0, 0, screen.width, screen.height)
        sceneview.scene = scene
        sceneview.delegate = delegate
        

        # set up the custom view controller
        methods = [CustomViewController_touchesBegan_withEvent_,            CustomViewController_viewWillAppear_, CustomViewController_viewWillDisappear_]
        protocols = []
        CustomViewController = create_objc_class('CustomViewController', UIViewController, methods=methods, protocols=protocols)
        cvc = CustomViewController.new().init().autorelease()
        cvc.view = sceneview
        

        # last set up
        self_objc = ObjCInstance(self)
        self_objc.addSubview_(sceneview)
 
        # workaround : I need to call manually viewWillAppear as otherwise my callback is not called...
        cvc.viewWillAppear_(False)
        

    def will_close(self):
        session = sceneview.session()
        session.pause()


@on_main_thread
def main():
    v = MyARView()
    v.present('full_screen', hide_title_bar=True)

if __name__ == '__main__':
    main()

I’ve created the following script allowing you to execute stash commands from your desktop.

If you are interested, have a look here :
https://github.com/Brun0oO/Pythonista/blob/master/rshell/rshell.py

And got some explanations, here :
https://github.com/ywangd/stash/issues/203

If you have a VR headset for your iOS device, you know it's difficult to display web VR content in full screen.
To try to solve this problem and to find a workaround, I created a tool.
If you are interested, have a look to my public GitHub here and explore the webVR project.

@JonB : Simple...Thank you!

Hi,

Is it normal : if I put a simple pythonista_startup.py in the site-packages folder with a simple (and single) print(« hello ») inside, I see « hello » twice in the console...
I would like to understand the reason because i’m using the dgelessus’s enabled_faulthandler script and it’s also called twice ....

Regards

@mikael said:

What is the ObjC exception?

Thank you a lot for the dgelessus's trick !
Here the detail of the caught exception :

Fatal Python error: Bus error

Thread 0x000000016e253000 (most recent call first):
  File "/var/containers/Bundle/Application/8FF92FC6-5C37-4D62-B81A-058EFF5E9BFF/Pythonista3.app/Frameworks/Py3Kit.framework/pylib/selectors.py", line 377 in select
  File "/var/containers/Bundle/Application/8FF92FC6-5C37-4D62-B81A-058EFF5E9BFF/Pythonista3.app/Frameworks/Py3Kit.framework/pylib/socketserver.py", line 237 in serve_forever
  File "/var/containers/Bundle/Application/8FF92FC6-5C37-4D62-B81A-058EFF5E9BFF/Pythonista3.app/Frameworks/Py3Kit.framework/pylib/site-packages/werkzeug/serving.py", line 437 in serve_forever
  File "/var/containers/Bundle/Application/8FF92FC6-5C37-4D62-B81A-058EFF5E9BFF/Pythonista3.app/Frameworks/Py3Kit.framework/pylib/site-packages/werkzeug/serving.py", line 693 in inner
  File "/var/containers/Bundle/Application/8FF92FC6-5C37-4D62-B81A-058EFF5E9BFF/Pythonista3.app/Frameworks/Py3Kit.framework/pylib/site-packages/werkzeug/serving.py", line 711 in run_simple
  File "/private/var/mobile/Containers/Shared/AppGroup/A0225C98-C19E-4DB5-B0EE-E750A7DDCC8E/Pythonista3/Documents/site-packages/flask/app.py", line 841 in run
  File "/private/var/mobile/Library/Mobile Documents/iCloud~com~omz-software~Pythonista3/Documents/webvr_embedded/main.py", line 152 in run
  File "/var/containers/Bundle/Application/8FF92FC6-5C37-4D62-B81A-058EFF5E9BFF/Pythonista3.app/Frameworks/Py3Kit.framework/pylib/threading.py", line 917 in _bootstrap_inner
  File "/var/containers/Bundle/Application/8FF92FC6-5C37-4D62-B81A-058EFF5E9BFF/Pythonista3.app/Frameworks/Py3Kit.framework/pylib/threading.py", line 885 in _bootstrap

Thread 0x000000016dcbb000 (most recent call first):
  File "/private/var/mobile/Library/Mobile Documents/iCloud~com~omz-software~Pythonista3/Documents/webvr_embedded/main.py", line 206 in __init__
  File "/private/var/mobile/Library/Mobile Documents/iCloud~com~omz-software~Pythonista3/Documents/webvr_embedded/main.py", line 258 in <module>

As you can see, I think the problem comes from my embedded http server. I do not have to close it properly (despite my attempt)... well it's not easy because of the external library and the multi threading mechanism used...I will try to look more closely (if anyone wants to help me, I'm interested ;o)

@mikael said:

Please do submit improvements or ideas, either here or as issues/pull requests.

Ok I made my first pull requests on your repository ;o)
do not hesitate to tell me if this is badly done...

I think I still have a kind of memory leak in my script since I can't run it twice (Pythonista leaves unexpectedly) but I think it's not related to your script...
Indeed strange thing : I have to call a blank page when closing my script otherwise the sound of my video continues to play...

I got it, now it works !
I'm going to suggest some modifications of wkwebview.py to its author ;o)

Note: I need to find a better way to clear the web cache... sometimes it fails...

@JonB : thanks for your advice but I think I host all the files locally and I use theses resources correctly (have a look to the 'web/static/js' directory and the 'vr_embedded.html' file). Is there anything in particular you're referring to?

Hi,

Here a demo : https://github.com/Brun0oO/Pythonista/tree/master/webvr_embedded

My goal is to embed a webvr experience using pythonista and more specially to display a stereoscopic 360 video (it sounds crazy does it not?).

My results :

* If I use a webview then I manage to display an online webvr experience.
* If I use a local flask server then I manage to stream a video to my webview.
* But if I try to embed the webvr experience and use my local streaming server then the same video is not displayed...

Can anyone help me ?

Thanks a lot !

Hi,
Finally, I managed to initialize an ar session
but I remain open to any improvement (
especially the famous ARKit constants and the call to the sleep function ;o)

Here the code :

# coding: utf-8
import objc_util

from objc_util import *
import ui
import os
import sys

#from myDebugToolKit import *
import time
from enum import IntFlag


load_framework('SceneKit')
load_framework('ARKit')

# Some 'constants' used by ARkit
# But i can't transfer them to the ARKit framework, why ?????

class ARWorldAlignment(IntFlag):
    ARWorldAlignmentGravity = 0
    ARWorldAlignmentGravityAndHeading = 1
    ARWorldAlignmentCamera = 2

class ARPlaneDetection(IntFlag):
    ARPlaneDetectionNone = 0
    ARPlaneDetectionHorizontal = 1 << 0
    ARPlaneDetectionVertical = 1 << 1

# Work In Progress here, I'm deciphering the ARKit constants...
#class ARSCNDebugOption(IntFlag):
#    ARSCNDebugOptionNone = 0
#    ARSCNDebugOptionShowWorldOrigin = int("ffffffff80000000", 16)
#    ARSCNDebugOptionShowFeaturePoints = int("ffffffff40000000", 16)

class ARSessionRunOptions(IntFlag):
    ARSessionRunOptionsNone                     = 0
    ARSessionRunOptionResetTracking             = 1 << 0
    ARSessionRunOptionRemoveExistingAnchors     = 1 << 1


NSError = ObjCClass('NSError')
SCNScene = ObjCClass('SCNScene')
ARSCNView = ObjCClass('ARSCNView')
ARWorldTrackingConfiguration = ObjCClass('ARWorldTrackingConfiguration')
ARSession = ObjCClass('ARSession')
UIViewController = ObjCClass('UIViewController')
ARPlaneAnchor = ObjCClass('ARPlaneAnchor')




# I should refactor te following line in a class but I need to learn more the create_objcc_class function
sceneview = None

# Here some set up functions used by the main class
def createSampleScene():
    # an empty scene
    scene = SCNScene.scene()
    return scene

def setDebugOptions(arscn):
    # Work In Progress Here, I'm trying to decipher the arkit constants...
    #val = ARSCNDebugOption.ARSCNDebugOptionShowWorldOrigin | ARSCNDebugOption.ARSCNDebugOptionShowFeaturePoints
    val = int("fffffffffc000000", 16) # this value is a combination of ShowWorldOrigin and ShowFeaturePoints flags, but I can't isolate each flags....
    print('Before calling setDebugOptions_(%s) : debugOptions=%s' %(hex(val), hex(arscn.debugOptions())))
    arscn.setDebugOptions_(val)
    print('After calling setDebugOptions_(%s) : debugOptions=%s' % (hex(val),hex(arscn.debugOptions())))


def createARSceneView(x, y, w, h, debug=True):
    v = ARSCNView.alloc().initWithFrame_((CGRect(CGPoint(x, y), CGSize(w, h))))
    v.setShowsStatistics_(debug) # I love statistics...
    return v

# Some callback definitions used by create_objc_class
def CustomViewController_touchesBegan_withEvent_(_self, _cmd, _touches, event):
    touches = ObjCInstance(_touches)
    for t in touches:
        loc = t.locationInView_(sceneview)
        sz = ui.get_screen_size()
        print(loc)

@on_main_thread
def runARSession(arsession):
    arconfiguration = ARWorldTrackingConfiguration.alloc().init()
    arconfiguration.setPlaneDetection_ (ARPlaneDetection.ARPlaneDetectionHorizontal)
    arconfiguration.setWorldAlignment_(ARWorldAlignment.ARWorldAlignmentGravity) # I do not use ARWorldAlignmentGravityAndHeading anymore because on my device, sometimes it fails to initialize the ar session because of an unitialized sensor (error 102). I think my magnetic phone casing plays tricks on me...

    arsession.runWithConfiguration_options_(arconfiguration, ARSessionRunOptions.ARSessionRunOptionResetTracking | ARSessionRunOptions.ARSessionRunOptionRemoveExistingAnchors )

    time.sleep(0.5) # Let the system breathe ;o) Ok, that's the workarround I found to retrieve the ar session configuration (otherwise I got None)....
    print('configuration',arsession.configuration()) # Very usefull for the debuging (at least for me !)


def CustomViewController_viewWillAppear_(_self, _cmd, animated):
    return

def CustomViewController_viewWillDisappear_(_self, _cmd, animated):
    session = sceneview.session()
    session.pause()

def MyARSCNViewDelegate_renderer_didAdd_for_(_self, _cmd, scenerenderer, node, anchor):
    if not isinstance(anchor, (ARPlaneAnchor)):
        return
    # to be implemented...


def MyARSCNViewDelegate_session_didFailWithError_(_self,_cmd,_session,_error):
    print('error',_error,_cmd,_session)
    err_obj=ObjCInstance(_error)
    print(err_obj) # Again, very usefull for the debuging...

# The main class...
class MyARView(ui.View):
    def __init__(self):
        super().__init__(self)


    @on_main_thread
    def initialize(self):
        global sceneview
        self.flex = 'WH'

        screen = ui.get_screen_size()

        # set up the scene
        scene = createSampleScene()

        # set up the ar scene view delegate
        methods = [MyARSCNViewDelegate_renderer_didAdd_for_,MyARSCNViewDelegate_session_didFailWithError_]
        protocols = ['ARSCNViewDelegate']
        MyARSCNViewDelegate = create_objc_class('MyARSCNViewDelegate', NSObject, methods=methods, protocols=protocols)
        delegate = MyARSCNViewDelegate.alloc().init()

        # set up the ar scene view
        sceneview = createARSceneView(0, 0, screen.width, screen.height)
        sceneview.scene = scene
        sceneview.setDelegate_(delegate)


        # set up the custom view controller
        methods = [CustomViewController_touchesBegan_withEvent_, CustomViewController_viewWillAppear_, CustomViewController_viewWillDisappear_]
        protocols = []
        CustomViewController = create_objc_class('CustomViewController', UIViewController, methods=methods, protocols=protocols)
        cvc = CustomViewController.alloc().init()
        cvc.view = sceneview


        # internal scheming...
        self_objc = ObjCInstance(self)
        self_objc.nextResponder().addChildViewController_(cvc)
        self_objc.addSubview_(sceneview)
        cvc.didMoveToParentViewController_(self_objc)

        # here, we try...
        runARSession(sceneview.session()) # I call here this function because I'm trying to find the best place to run the ar session...

        setDebugOptions(sceneview) # I call here this function because I'm trying to find the best place to set the debuging options....

    def will_close(self):
        session = sceneview.session()
        session.pause()



if __name__ == '__main__':
    v = MyARView()
    v.present('full_screen', hide_title_bar=True, orientations=['portrait'])
    v.initialize()

Note: if someone can correct the autorotate, I'm also interested !o)
Edit: I found a solution for my autorotate problem -> I turn it off ;o)

Hi all, stay tuned for this topic...

I've made some progress since the 3.3 beta delivery and some new personal investigations watching ARKit headers.

... I'm refactoring my basic code sample...

@mikael and @JonB, many thanks. It's better, now, the custom view controller is correctly "mounted".

I'm investigating the ARSession problem and i'm trying to understand why its configuration stays null after a "runWithConfiguration_" call (I'm going to use some delegate methods in order to spy the initialization process...).
I will update my github ASAP.

Stay tuned...