|
let myCanvas |
|
const arrows = [] |
|
let current_points = [] |
|
let t_size |
|
let button |
|
let isOnCanvas = false; |
|
let socket; |
|
let live = false; |
|
let timer; |
|
let capture; |
|
let instructions; |
|
let current_instructions; |
|
let messages_list = [] |
|
let timerWaiting; |
|
let waiting = false; |
|
|
|
let active_rearcam = false; |
|
let camConfig = "user"; |
|
let isOnMobile; |
|
let camIsON = false; |
|
|
|
function isMobileDevice() { |
|
return (typeof window.orientation !== "undefined") || (navigator.userAgent.indexOf('IEMobile') !== -1); |
|
} |
|
|
|
|
|
|
|
|
|
function preload(){ |
|
|
|
isOnMobile = isMobileDevice() |
|
console.log("is on mobile ? " + isOnMobile) |
|
|
|
|
|
socket = io(); |
|
|
|
socket.on("connect", () => { |
|
console.log(socket.id); |
|
}); |
|
|
|
socket.on("hello", (arg) => { |
|
let api_data_test = arg[1] |
|
console.log(api_data_test) |
|
}); |
|
|
|
socket.on("api_error", (arg) => { |
|
let message = arg |
|
console.log(message) |
|
}); |
|
|
|
} |
|
|
|
function call_api(socket, instructions){ |
|
console.log("Calling API ... ") |
|
waiting = true; |
|
show_loading() |
|
cnv_data = myCanvas.elt.toDataURL('image/png'); |
|
let data = [ |
|
cnv_data, |
|
instructions |
|
] |
|
socket.emit("ask_api", data); |
|
} |
|
|
|
function windowResized() { |
|
if(windowWidth <= 500){ |
|
resizeCanvas(380, 380*320/576); |
|
} else { |
|
resizeCanvas(576, 320); |
|
} |
|
} |
|
|
|
function setup() { |
|
pixelDensity(1) |
|
if(windowWidth < 500){ |
|
myCanvas = createCanvas(380, 380*320/576); |
|
} else { |
|
myCanvas = createCanvas(576, 320); |
|
} |
|
background(220) |
|
myCanvas.id('myCanvas') |
|
myCanvas.parent("canvas-container") |
|
|
|
socket.on("api_response", (response) => { |
|
waiting = false; |
|
clearTimeout(timerWaiting) |
|
|
|
|
|
vision_text = response[0] |
|
console.log(vision_text) |
|
messages_list.push(vision_text) |
|
|
|
if(live === true){ |
|
loadingDiv = document.getElementById("loading-div") |
|
loadingDiv.innerHTML = "_" |
|
display_messages() |
|
console.log("Sending new requestion in 3 seconds") |
|
|
|
timer = setTimeout(() => { |
|
console.log(current_instructions) |
|
call_api(socket, current_instructions) |
|
}, 3000) |
|
} |
|
}); |
|
|
|
|
|
let getCanvas = document.getElementById('myCanvas'); |
|
|
|
getCanvas.addEventListener("pointerdown", (e) => { |
|
|
|
getCanvas.setPointerCapture(e.pointerId); |
|
isOnCanvas = true; |
|
}, false); |
|
|
|
getCanvas.addEventListener("pointerup", (e) => { |
|
|
|
|
|
if (isOnCanvas) { |
|
getCanvas.releasePointerCapture(e.pointerId); |
|
isOnCanvas = false; |
|
} |
|
}, false); |
|
|
|
|
|
|
|
start_capture_button=createButton("turn on cam") |
|
start_capture_button.mousePressed(() => { |
|
camIsON = true; |
|
if(!active_rearcam){ |
|
camConfig = "user" |
|
} else { |
|
camConfig = { |
|
exact: "environment" |
|
} |
|
} |
|
let constraints = { |
|
audio: false, |
|
video: { |
|
facingMode: camConfig |
|
} |
|
}; |
|
capture = createCapture(constraints); |
|
capture.hide(); |
|
}) |
|
|
|
if(isOnMobile){ |
|
checkbox_rearcam = createCheckbox("switch to rear cam"); |
|
checkbox_rearcam.parent("checkbox-rear"); |
|
|
|
checkbox_rearcam.mousePressed( () => { |
|
if(active_rearcam == false){ |
|
active_rearcam = true; |
|
} else { |
|
active_rearcam = false |
|
} |
|
|
|
if(camIsON){ |
|
if(capture){ |
|
capture.remove() |
|
capture = undefined |
|
} |
|
|
|
if(active_rearcam == true){ |
|
let new_constraints = { |
|
audio: false, |
|
video: { |
|
facingMode: { |
|
exact: "environment" |
|
} |
|
} |
|
}; |
|
capture = createCapture(new_constraints) |
|
capture.hide() |
|
} else { |
|
let new_constraints = { |
|
audio: false, |
|
video: { |
|
facingMode: "user" |
|
} |
|
|
|
|
|
|
|
}; |
|
capture = createCapture(new_constraints) |
|
capture.hide() |
|
} |
|
} |
|
}) |
|
|
|
} |
|
|
|
start_button = createButton('start live vision'); |
|
start_button.mousePressed(() => { |
|
live = true; |
|
current_instructions = input_instructions.value() |
|
console.log("Live in ON") |
|
console.log(current_instructions) |
|
call_api(socket, current_instructions) |
|
}) |
|
|
|
stop_button = createButton('stop all'); |
|
stop_button.mousePressed(() => { |
|
live = false; |
|
waiting = false; |
|
|
|
clearTimeout(timer); |
|
clearTimeout(timerWaiting) |
|
loadingDiv.innerHTML = "_" |
|
if(capture){ |
|
capture.remove(); |
|
capture = undefined |
|
camIsON = false; |
|
} |
|
console.log("live is OFF") |
|
|
|
}) |
|
|
|
input_instructions = createInput("What is happening ? ") |
|
|
|
change_instructions = createButton("change instructions") |
|
change_instructions.mousePressed(() => { |
|
current_instructions = input_instructions.value() |
|
}) |
|
|
|
start_capture_button.parent("buttons-container") |
|
start_button.parent("buttons-container") |
|
stop_button.parent("buttons-container") |
|
input_instructions.parent("instructions-container") |
|
change_instructions.parent("instructions-container") |
|
|
|
loadingDiv = document.getElementById("loading-div") |
|
|
|
} |
|
|
|
function draw() { |
|
background(220); |
|
textAlign(CENTER); |
|
text('turn on your webcam', width/2, height/2); |
|
|
|
if(capture != undefined){ |
|
if(!active_rearcam){ |
|
|
|
translate(capture.width, 0); |
|
|
|
|
|
scale(-1, 1); |
|
} |
|
imageMode(CENTER); |
|
image(capture, 0, 0) |
|
} |
|
} |
|
|
|
function display_messages(){ |
|
visionDiv = document.getElementById("vision-text-container") |
|
visionDiv.innerHTML = "" |
|
|
|
|
|
|
|
|
|
|
|
|
|
newNode_last = document.createElement('div'); |
|
newNode_last.classList.add('text-msg') |
|
visionDiv.appendChild(newNode_last); |
|
const text_msg = messages_list[messages_list.length - 1] |
|
const words = text_msg.split(" "); |
|
|
|
|
|
function wait(ms) { |
|
return new Promise((resolve) => setTimeout(resolve, ms)); |
|
} |
|
|
|
async function streamText(words) { |
|
for (let word of words) { |
|
|
|
newNode_last.innerHTML += " " + word; |
|
|
|
await wait(30); |
|
} |
|
} |
|
|
|
streamText(words); |
|
|
|
} |
|
|
|
function clean(){ |
|
if(waiting == true){ |
|
loadingDiv.innerHTML = "" |
|
timerWaiting = setTimeout(show_loading, 500) |
|
} else { |
|
loadingDiv.innerHTML = "_" |
|
} |
|
|
|
} |
|
|
|
function show_loading(){ |
|
|
|
if(waiting == true){ |
|
loadingDiv.innerHTML = "" |
|
loading_text = ". . ." |
|
} else { |
|
loading_text = "_" |
|
} |
|
|
|
const dots = loading_text.split(" "); |
|
|
|
|
|
function wait(ms) { |
|
if(waiting == true){ |
|
|
|
return new Promise((resolve) => resolveTimer = setTimeout(resolve, ms)); |
|
} else { |
|
clearTimeout(resolveTimer) |
|
loadingDiv.innerHTML = "_" |
|
} |
|
|
|
} |
|
|
|
async function streamDots(words) { |
|
|
|
if(waiting == true){ |
|
|
|
for (let word of words) { |
|
|
|
loadingDiv.innerHTML += " " + word; |
|
|
|
await wait(500); |
|
} |
|
|
|
|
|
|
|
|
|
|
|
clean() |
|
} else { |
|
clearTimeout(timerWaiting) |
|
|
|
} |
|
} |
|
|
|
streamDots(dots); |
|
|
|
|
|
} |
|
|
|
|
|
|