Skip to content

Instantly share code, notes, and snippets.

@wizard1066
Created February 21, 2023 17:13
Show Gist options
  • Save wizard1066/d93a994642268e3453ac28e1a0eff318 to your computer and use it in GitHub Desktop.
Save wizard1066/d93a994642268e3453ac28e1a0eff318 to your computer and use it in GitHub Desktop.
import SwiftUI
import SceneKit
import ARKit
import Combine
let actionPublisher = PassthroughSubject<Int,Never>()
var actionSubscriber:AnyCancellable!
var setSquare = PassthroughSubject<Int,Never>()
let tts = AVSpeechSynthesizer()
struct Fonts {
static func avenirNextCondensedBold (size: CGFloat) -> Font {
return Font.custom("AvenirNextCondensed-Bold", size: size)
}
static func neutonRegular (size: CGFloat) -> Font {
return Font.custom("Neuton-Regular", size: size)
}
}
enum Positions {
case preprepos
case prepos
case thepos
case postpos
case postpostpos
}
struct pickerValues {
var id = UUID()
var value:String = ""
init(id: UUID = UUID(), value: String) {
self.id = id
self.value = value
}
}
let vowels = ["e","t","a","o","i","n","s","r","h","d","l","u","c","m","f","y","w","g","p","b","v","k","x","q","j","z"]
//let vowels = ["1","2","3","4","5","6","7"]
var talk = Speaking()
struct ContentView: View {
@State var arview = ARSCNView()
@ObservedObject private var looked = Looker.shared
private var looks = Looks.shared
var body: some View {
ZStack {
CustomARView(view: arview)
VStack {
//VocabDetailView()
Text("pause \(looked.paused.description)")
.font(Fonts.avenirNextCondensedBold(size: 24))
.foregroundColor(looked.paused ? Color.green: Color.red)
.background(Color.clear)
Text("spell \(looked.spell.description)")
.font(Fonts.avenirNextCondensedBold(size: 24))
.foregroundColor(looked.spell ? Color.green: Color.red)
.background(Color.clear)
Text("count \(looked.allWords.count)")
.font(Fonts.avenirNextCondensedBold(size: 24))
.foregroundColor(looked.spell ? Color.blue: Color.blue)
.background(Color.clear)
Text(looked.sentence)
.font(Fonts.avenirNextCondensedBold(size: 48))
.foregroundColor(Color.black)
.frame(height: 48)
.onAppear(perform: {
for word in vowels {
looked.allWords.append(word)
}
looked.isReady = true
})
if looked.isReady {
HStack {
Text(newPicker(position:.postpos))
.font(Fonts.avenirNextCondensedBold(size: 32))
.foregroundColor(Color.white.opacity(0.7))
Text(newPicker(position:.thepos))
.font(Fonts.avenirNextCondensedBold(size: 48))
.foregroundColor(Color.white)
Text(newPicker(position:.prepos))
.font(Fonts.avenirNextCondensedBold(size: 32))
.foregroundColor(Color.white.opacity(0.7))
}
// Add your squares in here
}
Spacer()
Text(looked.words)
.font(Fonts.avenirNextCondensedBold(size: 48))
.foregroundColor(Color.green)
.frame(height: 48)
.background(Color.red)
HStack {
Text("\(looked.pVowel3)")
.font(Fonts.avenirNextCondensedBold(size: 64))
.foregroundColor(Color.white.opacity(0.4))
Text("\(looked.pVowel2)")
.font(Fonts.avenirNextCondensedBold(size: 64))
.foregroundColor(Color.white.opacity(0.6))
Text("\(looked.pVowel1)")
.font(Fonts.avenirNextCondensedBold(size: 64))
.foregroundColor(Color.white.opacity(0.8))
Text("\(looked.vowel)")
.font(Fonts.avenirNextCondensedBold(size: 96))
.foregroundColor(Color.white)
Text("\(looked.nVowel3)")
.font(Fonts.avenirNextCondensedBold(size: 64))
.foregroundColor(Color.white.opacity(0.8))
Text("\(looked.nVowel2)")
.font(Fonts.avenirNextCondensedBold(size: 64))
.foregroundColor(Color.white.opacity(0.6))
Text("\(looked.nVowel1)")
.font(Fonts.avenirNextCondensedBold(size: 64))
.foregroundColor(Color.white.opacity(0.4))
}.frame(height: 80)
}
}.onAppear(perform: {
DispatchQueue.main.asyncAfter(deadline: .now() + 2) {
tts.speak(AVSpeechUtterance(string: "hello"))
loop = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { timer in
//print("repeating \(timer) \(self.looked.autoRight) \(self.looked.autoLeft)")
// if self.looked.autoRight {
// Task { await self.looks.addShapes(faceSeen: 0.1) }
// }
// if self.looked.autoLeft {
// Task { await self.looks.addShapes(faceSeen: -0.1) }
// }
}
}
})
}
func newPicker(position: Positions) -> String {
if looked.allWords.isEmpty {
return ""
}
if position == .postpos {
if looked.selectedWord > 1 {
return self.looked.allWords[looked.selectedWord - 1]
}
}
if position == .thepos {
if looked.selectedWord > 0 {
return self.looked.allWords[looked.selectedWord]
}
}
if position == .prepos {
if looked.selectedWord < looked.allWords.count - 1 {
return self.looked.allWords[looked.selectedWord + 1]
}
}
return ""
}
}
struct VocabDetailView: View {
let synth = AVSpeechSynthesizer()
private func readOut(text: String) {
let utterance = AVSpeechUtterance(string: text)
utterance.voice = AVSpeechSynthesisVoice(language: "en-US")
synth.speak(utterance)
}
var body: some View {
HStack{
Button("Play") {
readOut(text: "test")
}
}
}
}
class Looker: ObservableObject {
static var shared = Looker()
@Published var angleX:Float = 0.0
@Published var angleY:Float = 0.0
@Published var angleZ:Float = 0.0
@Published var paused:Bool = false
@Published var gazeY:Float = 0.0
@Published var gazeX:Float = 0.0
@Published var gazeZ:Float = 0.0
@Published var pVowel1 = ""
@Published var pVowel2 = ""
@Published var pVowel3 = ""
@Published var vowel:String = ""
@Published var nVowel1 = ""
@Published var nVowel2 = ""
@Published var nVowel3 = ""
@Published var vindex = 0
@Published var words = ""
@Published var outOfBounds = false
@Published var saved:Float = 0
@Published var selectedWord = 0
@Published var allWords:[String] = []
@Published var isReady = false
@Published var spell = true
@Published var sentence = ""
@Published var autoRight = false
@Published var autoLeft = false
}
actor Looks: NSObject {
static var shared = Looks()
var gazesX:[Float] = []
var gazesY:[Float] = []
var faces:[Double] = []
var facesUp:[Double] = []
func returnInner() -> Int {
//defer { faces.removeAll() }
return faces.filter{ $0 > 0.1 }.count
}
func facesClear() {
faces.removeAll()
}
func addFace(face:Double) {
faces.append(face)
}
func returnFaceUp() -> Int {
return facesUp.filter{ $0 > 0.1 }.count
}
func facesUpClear() {
facesUp.removeAll()
}
func addFaceUp(face:Double) {
facesUp.append(face)
}
func addShades(faceSeen:Float) {
for _ in 0..<24 {
gazesY.append(faceSeen)
}
}
func addShade(faceSeen:Float) {
gazesY.append(faceSeen)
}
func addShapes(faceSeen:Float) {
for _ in 0..<24 {
gazesX.append(faceSeen)
}
}
func addShape(faceSeen:Float) {
gazesX.append(faceSeen)
}
func upShades() -> Int {
let gazesSeen = gazesY.filter( { $0 > 0 } )
return gazesSeen.count
}
func downShades() -> Int {
let gazesSeen = gazesY.filter( { $0 < 0 } )
return gazesSeen.count
}
func rightShapes() -> Int {
let gazesSeen = gazesX.filter( { $0 > 0 } )
return gazesSeen.count
}
func leftShapes() -> Int {
let gazesSeen = gazesX.filter( { $0 < 0 } )
return gazesSeen.count
}
func resetShapes() {
gazesX.removeAll()
gazesY.removeAll()
}
}
struct CustomARView: UIViewRepresentable {
typealias UIViewType = ARSCNView
var view:ARSCNView
var options: [Any] = []
func makeUIView(context: Context) -> ARSCNView {
view.session.delegate = context.coordinator
return view
}
func updateUIView(_ view: ARSCNView, context: Context) {
}
func makeCoordinator() -> Coordinator {
Coordinator(self.view)
}
}
var loop:Timer!
class Coordinator: NSObject, ARSCNViewDelegate, ARSessionDelegate {
private var trackingView:ARSCNView
private var sphereNode: SCNNode!
private var cubeNode: SCNNode!
private var cubeNode2: SCNNode!
private var textGeo: SCNText!
private var textNode: SCNNode!
private var textGeo2: SCNText!
private var textNode2: SCNNode!
private var faceGeometry: ARSCNFaceGeometry!
private var faceNode: SCNNode!
private var spawnTime:TimeInterval = 0
private var lastTime:TimeInterval? = nil
private var faceAnchor: ARFaceAnchor? = nil
private var connected = MultipeerSession()
private var lastWord: String = ""
private var looks = Looks.shared
private var bucket: Int = 0
@ObservedObject private var looked = Looker.shared
init(_ view: ARSCNView) {
self.trackingView = view
super.init()
guard ARFaceTrackingConfiguration.isSupported else {
fatalError("Face tracking not available on this on this device model!")
}
let configuration = ARFaceTrackingConfiguration()
self.trackingView.session.run(configuration)
self.trackingView.delegate = self
let geo3 = SCNSphere(radius: 0.01)
geo3.segmentCount = 16
sphereNode = SCNNode(geometry: geo3)
sphereNode.geometry?.firstMaterial?.diffuse.contents = UIColor.white.withAlphaComponent(0.99)
sphereNode.geometry?.firstMaterial?.fillMode = .fill
let colorNames:[UIColor] = [UIColor.red, .blue, .green, .purple, .orange, .brown]
var colorMaterials:[SCNMaterial] = []
for colors in colorNames {
let newMaterial = SCNMaterial()
newMaterial.diffuse.contents = colors
colorMaterials.append(newMaterial)
}
let geo2 = SCNBox(width: 0.020, height: 0.020, length: 0.020, chamferRadius: 0.1)
cubeNode = SCNNode(geometry: geo2)
cubeNode.geometry?.materials = colorMaterials
cubeNode2 = SCNNode(geometry: geo2)
cubeNode2.geometry?.materials = colorMaterials
let textGeo = SCNText(string: "down", extrusionDepth: CGFloat(1 / 10))
let font = UIFont(name: "Neuton-Regular", size: 1.0)
textGeo.font = font
textGeo.firstMaterial!.diffuse.contents = UIColor.black
textGeo.firstMaterial?.fillMode = .fill
textGeo.alignmentMode = CATextLayerAlignmentMode.center.rawValue
textGeo.chamferRadius = 0.01
let (minBound, maxBound) = textGeo.boundingBox
textNode = SCNNode(geometry: textGeo)
textNode.pivot = SCNMatrix4MakeTranslation( (maxBound.x - minBound.x)/2 , maxBound.y , -0.5)
textNode.scale = SCNVector3Make(0.05, 0.05, 0.05)
let textGeo2 = SCNText(string: "up", extrusionDepth: CGFloat(1 / 10))
textGeo2.font = font
textGeo2.firstMaterial!.diffuse.contents = UIColor.black
textGeo2.firstMaterial?.fillMode = .fill
textGeo2.alignmentMode = CATextLayerAlignmentMode.center.rawValue
textGeo2.chamferRadius = 0.01
let (minBound2, maxBound2) = textGeo2.boundingBox
textNode2 = SCNNode(geometry: textGeo2)
textNode2.pivot = SCNMatrix4MakeTranslation( (maxBound2.x - minBound2.x)/2 , maxBound2.y , -0.5)
textNode2.scale = SCNVector3Make(0.05, 0.05, 0.05)
}
var counter1 = 1
var counter2 = -1
var maxBuz:Float = 0
var minBuz:Float = 0
func renderer(_ renderer: SCNSceneRenderer, nodeFor anchor: ARAnchor) -> SCNNode? {
let device = trackingView.device
faceGeometry = ARSCNFaceGeometry(device: device!)
faceNode = SCNNode(geometry: faceGeometry)
faceNode.geometry?.firstMaterial?.fillMode = .lines
faceNode.geometry?.firstMaterial?.diffuse.contents = UIColor.white.withAlphaComponent(0.75)
faceNode.addChildNode(sphereNode)
cubeNode.simdPosition.z += 0.2
return faceNode
}
var spoke:String = ""
func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) {
if time > spawnTime {
Task {
let looksRight = await looks.rightShapes()
let looksLeft = await looks.leftShapes()
let looksUp = await looks.upShades()
let browCount = await looks.returnInner()
let faceUp = await looks.returnFaceUp()
print("brows \(browCount)")
if browCount > 45 {
DispatchQueue.main.async { [self] in
self.looked.paused = true
Task { await looks.facesClear() }
if !looked.words.isEmpty {
looked.words.removeLast()
looked.paused = true
} else {
if !looked.sentence.isEmpty {
let foos = looked.sentence.components(separatedBy: "#")
let nfoos = foos.dropLast(2)
let nfoo = nfoos.joined(separator: "#")
looked.sentence = nfoo
if nfoo != "" {
looked.sentence += "#"
}
}
}
DispatchQueue.main.asyncAfter(deadline: .now() + 2) { [self] in
self.looked.paused = false
}
}
}
if looksRight > 16 {
looked.autoLeft = false
looked.autoRight = false
self.looked.vindex += 1
self.looked.vindex = self.looked.vindex % vowels.count
}
if looksLeft > 16 {
looked.autoRight = false
looked.autoLeft = false
self.looked.vindex -= 1
if self.looked.vindex < 0 {
self.looked.vindex = vowels.count - 1
}
}
// if looksRight > 32 {
// looked.autoLeft = false
// looked.autoRight = true
// }
// if looksLeft > 32 {
// looked.autoLeft = true
// looked.autoRight = false
// }
if faceUp > 32 {
print("facesUp \(faceUp)")
DispatchQueue.main.async { [self] in
Task { await looks.facesUpClear() }
print("looksup \(looksUp)")
looked.paused = true
looked.spell.toggle()
var unfound = true
for values in looked.allWords {
if looked.words == values {
unfound = false
}
}
if unfound {
if looked.sentence.count > 0 && looked.words.count > 1 {
looked.sentence += looked.words
looked.allWords.append(looked.words)
looked.words = ""
if looked.sentence.last != "#" {
looked.sentence += "#"
}
}
}
looked.paused = false
}
}
await looks.resetShapes()
if (looksLeft != 0 || looksRight != 0) {
self.looked.vowel = vowels[self.looked.vindex]
if spoke != vowels[self.looked.vindex] {
talk.speaker(words: [vowels[self.looked.vindex]], speed: Speaking.Rate.fast)
spoke = vowels[self.looked.vindex]
}
if looked.vindex > 0 {
self.looked.pVowel1 = vowels[self.looked.vindex - 1]
} else {
self.looked.pVowel1 = ""
}
if looked.vindex > 1 {
self.looked.pVowel2 = vowels[self.looked.vindex - 2]
} else {
self.looked.pVowel2 = ""
}
if looked.vindex > 2 {
self.looked.pVowel3 = vowels[self.looked.vindex - 3]
} else {
self.looked.pVowel3 = ""
}
if looked.vindex < vowels.count - 1 {
self.looked.nVowel3 = vowels[self.looked.vindex + 1]
} else {
self.looked.nVowel3 = ""
}
if looked.vindex < vowels.count - 2 {
self.looked.nVowel2 = vowels[self.looked.vindex + 2]
} else {
self.looked.nVowel2 = ""
}
if looked.vindex < vowels.count - 3 {
self.looked.nVowel1 = vowels[self.looked.vindex + 3]
} else {
self.looked.nVowel1 = ""
}
self.looked.paused = false
var alphas:[Character] = []
var d:[Double] = []
for alpha in 97...97+25 {
alphas.append(Character(UnicodeScalar(alpha)!))
if looked.vowel.contains(Character(UnicodeScalar(alpha)!)) {
d.append(1)
} else {
d.append(0)
}
}
looked.isReady = false
looked.allWords.removeAll()
looked.selectedWord = 0
do {
let config = MLModelConfiguration()
let model = try word1Classifier(configuration: config)
let prediction = try model.prediction(a: d[0], b: d[1], c: d[2], d: d[3], e: d[4], f: d[5], g: d[6], h: d[7], i: d[8], j: d[9], k: d[10], l: d[11], m: d[12], n: d[13], o: d[14], p: d[15], q: d[16], r: d[17], s: d[18], t: d[19], u: d[20], v: d[21], w: d[22], x: d[23], y: d[24], z: d[25])
let sortedWords = prediction.wordProbability.sorted(by: {$0.value > $1.value })
for probs in sortedWords {
if #available(iOS 16.0, *) {
if probs.value > 0.012 && probs.key.contains(try! Regex("^\(looked.vowel)")) {
looked.allWords.append(probs.key)
}
} else {
// Fallback on earlier versions
}
}
} catch {
print("error")
}
looked.isReady = true
}
}
spawnTime = time + TimeInterval(1)
}
}
func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor) {
guard let faceAnchor = anchor as? ARFaceAnchor else { return }
//func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) {
// lastTime = time
// if time > spawnTime {
Task { if faceAnchor.blendShapes[.browInnerUp]!.doubleValue > 0.5 {
await looks.addFace(face: faceAnchor.blendShapes[.browInnerUp]!.doubleValue)
}}
Task { if faceAnchor.blendShapes[.eyeLookUpLeft]!.doubleValue > 0.2 {
await looks.addFaceUp(face: faceAnchor.blendShapes[.eyeLookUpLeft]!.doubleValue)
}}
if (looked.gazeX > 0.05 && !looked.paused && !looked.outOfBounds && looked.spell) {
DispatchQueue.main.async { [self] in
Task { await looks.addShape(faceSeen: looked.gazeX) }
}
}
if (looked.gazeX < -0.05 && !looked.paused && !looked.outOfBounds && looked.spell) {
DispatchQueue.main.async { [self] in
Task { await looks.addShape(faceSeen: looked.gazeX) }
}
}
faceGeometry.update(from: faceAnchor.geometry)
DispatchQueue.main.async { [self] in
let oOB = abs(looked.angleY)
if oOB < 16 {
looked.outOfBounds = false
} else {
looked.outOfBounds = true
}
}
//let foo = self.faceAnchor!.geometry.vertices[9]
//let newPos = [foo].reduce(vector_float3(), +) / Float([foo].count)
cubeNode.simdOrientation = faceNode.simdOrientation
cubeNode2.simdOrientation = faceNode.simdOrientation
DispatchQueue.main.async { [self] in
looked.gazeX = Float(faceAnchor.lookAtPoint.x)
looked.gazeY = Float(faceAnchor.lookAtPoint.y)
looked.angleX = GLKMathRadiansToDegrees(faceNode.eulerAngles.x)
looked.angleY = GLKMathRadiansToDegrees(faceNode.eulerAngles.y)
looked.angleZ = GLKMathRadiansToDegrees(faceNode.eulerAngles.z)
looked.saved = cubeNode.simdWorldPosition.x
}
if (faceAnchor.blendShapes[.tongueOut]!.doubleValue > 0.6 && !looked.paused && !looked.outOfBounds && looked.spell) {
DispatchQueue.main.async { [self] in
looked.paused = true
looked.words = looked.words + looked.vowel
DispatchQueue.main.asyncAfter(deadline: .now() + 2) { [self] in
self.looked.paused = false
lastWord = looked.words
talk.speaker(words: [looked.words], speed: Speaking.Rate.slow)
}
}
}
if (faceAnchor.blendShapes[.tongueOut]!.doubleValue > 0.6 && !looked.paused && !looked.outOfBounds && !looked.spell) {
DispatchQueue.main.async { [self] in
looked.paused = true
if looked.sentence == "" {
looked.sentence = looked.allWords[looked.selectedWord] + "#"
} else {
looked.sentence = looked.sentence + looked.allWords[looked.selectedWord]
if looked.sentence.last != "#" {
looked.sentence += "#"
}
}
DispatchQueue.main.asyncAfter(deadline: .now() + 2) { [self] in
self.looked.paused = false
talk.speaker(words: [looked.sentence], speed: Speaking.Rate.slow)
}
}
}
if (looked.gazeX < -0.05 && !looked.paused && !looked.outOfBounds && !looked.spell) {
DispatchQueue.main.async { [self] in
self.looked.paused = true
looked.selectedWord += 1
if looked.selectedWord > looked.allWords.count - 1 {
looked.selectedWord = 0
}
DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
self.looked.paused = false
}
}
}
if (looked.gazeX > 0.05 && !looked.paused && !looked.outOfBounds && !looked.spell) {
DispatchQueue.main.async { [self] in
self.looked.paused = true
looked.selectedWord -= 1
if looked.selectedWord < 0 {
looked.selectedWord = looked.allWords.count - 1
}
DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
self.looked.paused = false
}
}
}
}
}
struct SquareView: View {
@State var index: Int
@State var outline = true
var body: some View {
Rectangle()
.fill(Color.white)
.frame(width: 20, height: 20)
.onReceive(setSquare) { indexOf in
if indexOf == index {
outline.toggle()
}
}
}
}
struct ContentView_Previews: PreviewProvider {
static var previews: some View {
ContentView()
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment