app.py

import re
import base64
import datetime
import sys
import pickle
import gzip
import json
import time
import gradio as gr

from ai import *
from tricks import *
from decompositioner import *
from rendering import *


import os
import glob
import shutil

splash = glob.glob('ui/web-mobile/splash*')[0]
os.remove(splash)
shutil.copy('res/splash.png', splash)
with open('ui/web-mobile/index.html', 'r', encoding='utf-8') as f:
    page = f.read()
with open('ui/web-mobile/index.html', 'w', encoding='utf-8') as f:
    f.write(page.replace('Cocos Creator | ', ''))


def cv2_encode(image: np.ndarray, name):
    if image is None:
        return 'null'
    if '.jpg' in name:
        _, data = cv2.imencode('.jpeg', image)
        return 'data:image/jpeg;base64,' + base64.b64encode(data).decode('utf8')
    else:
        _, data = cv2.imencode('.png', image)
        return 'data:image/png;base64,' + base64.b64encode(data).decode('utf8')


def get_request_image(request, name):
    img = request.get(name)
    img = re.sub('^data:image/.+;base64,', '', img)
    img = base64.b64decode(img)
    img = np.fromstring(img, dtype=np.uint8)
    img = cv2.imdecode(img, -1)
    return img


def npcache(history, room_id, name, data):
    rooms = list(filter(lambda _room: _room["id"] == room_id, history))
    if len(rooms) == 0:
        room = { "id": room_id }
        history.append(room)
    else:
        room = rooms[0]
    room[name] = data


def npread(history, room_id, name):
    rooms = list(filter(lambda _room: _room["id"] == room_id, history))
    if len(rooms) == 0:
        return None
    else:
        room = rooms[0]
    return room.get(name, None)


def upload_sketch(json_str, history):
    request = json.loads(json_str)
    timenow = time.time()
    ID = datetime.datetime.now().strftime('H%HM%MS%S')
    room = datetime.datetime.now().strftime('%b%dH%HM%MS%S') + 'R' + str(np.random.randint(100, 999))
    sketch = from_png_to_jpg(get_request_image(request, 'sketch'))
    npcache(history, room, 'sketch.original.jpg', sketch)
    sketch = go_tail(cli_norm(min_resize(sketch, 512)))
    print('original_sketch saved')
    s256 = go_vector(go_cal(mk_resize(sketch, 8)))[:, :, 0]
    print('s256')
    s512 = go_vector(go_cal(d_resize(sketch, s256.shape, 2.0)))[:, :, 0]
    print('s512')
    s1024 = go_vector(go_cal(d_resize(sketch, s256.shape, 4.0)))[:, :, 0]
    print('s1024')
    npcache(history, room, 'sketch.s1024.png', s1024)
    npcache(history, room, 'sketch.s512.png', s512)
    npcache(history, room, 'sketch.s256.png', s256)
    print('edge processed')
    fill = double_fill(s1024, s512, s256)
    npcache(history, room, 'sketch.fill', fill)
    print('filled')
    npcache(history, room, 'sketch.colorization.png', np.min(sketch, axis=2))
    print('sketch processed')
    print(time.time() - timenow)
    if len(history) > 5:
        history = history[-5:]
    return room + '_' + ID, history


def request_result(json_str, history):
    request = json.loads(json_str)
    timenow = time.time()
    room = request.get("room")
    if len(list(filter(lambda _room: _room["id"] == room, history))) == 0:
        return None, history
    skipper = str(request.get("skipper"))
    light_r = float(request.get("r"))
    light_g = float(request.get("g"))
    light_b = float(request.get("b"))
    light_h = float(request.get("h"))
    light_max = max([light_r, light_g, light_b, light_h])
    inv4 = int(request.get("inv4"))
    print('inv4=' + str(inv4))
    light_r = (light_r + 1e-5) / (light_max + 1e-5)
    light_g = (light_g + 1e-5) / (light_max + 1e-5)
    light_b = (light_b + 1e-5) / (light_max + 1e-5)
    light_h *= 600.0
    light_d = request.get("d")
    need_render = int(request.get("need_render"))
    print([light_r, light_g, light_b, light_d])
    ID = datetime.datetime.now().strftime('H%HM%MS%S')
    points = request.get("points")
    points = json.loads(points)
    npcache(history, room, 'points.' + ID + '.txt', points)
    if len(points) > 500:
        return None, history
    for _ in range(len(points)):
        points[_][1] = 1 - points[_][1]
    fill = npread(history, room, 'sketch.fill')
    s1024 = npread(history, room, 'sketch.s1024.png')
    sketch = npread(history, room, 'sketch.colorization.png')
    print(skipper)
    if npread(history, room, 'albedo.' + skipper + '.png') is not None:
        albedo = npread(history, room, 'albedo.' + skipper + '.png')
        npcache(history, room, 'albedo.' + ID + '.png', albedo)
        print('albedo readed')
    else:
        faceID = int(request.get("faceID")) - 65535
        print(faceID)
        if faceID > -1:
            print('fake ref')
            face = from_png_to_jpg(cv2.imread("refs/" + str(faceID + 1) + ".png", cv2.IMREAD_UNCHANGED))
        else:
            print('get ref')
            face = from_png_to_jpg(get_request_image(request, 'face'))
        npcache(history, room, 'face.' + ID + '.jpg', face)
        face = s_enhance(face)
        print('request result room = ' + str(room) + ', ID = ' + str(ID))
        print('processing painting in ' + room)
        if inv4 > 0:
            sketch_1024 = k_resize(sketch, 64)
        else:
            sketch_1024 = k_resize(sketch, 48)
        hints_1024 = ini_hint(sketch_1024)
        hints_1024 = opreate_normal_hint(hints_1024, points, length=2, skip_sp=True)
        baby = go_head(
            sketch=sketch_1024,
            global_hint=k_resize(face, 14),
            local_hint=hints_1024
        )
        npcache(history, room, 'baby.' + ID + '.jpg', baby)
        print('baby born')
        composition = d_resize(re_deatlize(deatlize(balance_fill(baby, fill, opreate_normal_hint(ini_hint(s1024), points, length=2, skip_sp=True), s1024)), s1024), sketch.shape)
        npcache(history, room, 'composition.' + ID + '.jpg', composition)
        gird = process_overlay(composition, sketch)
        npcache(history, room, 'gird.' + ID + '.jpg', gird)
        print('composition saved')
        if inv4 > 0:
            albedo = go_render(sketch_1024, d_resize(composition, sketch_1024.shape, 0.5), hints_1024)
            albedo = go_tail(albedo)
            albedo = d_resize(re_deatlize(d_resize(albedo, s1024.shape), s1024), sketch.shape)
            albedo = cv2.cvtColor(albedo, cv2.COLOR_RGB2YUV)
            albedo[:, :, 0] = go_vgg7(albedo[:, :, 0])
            albedo = cv2.cvtColor(albedo, cv2.COLOR_YUV2RGB)
        else:
            albedo = re_deatlize(d_resize(baby, s1024.shape), s1024)
            albedo = d_resize(albedo, sketch.shape, 0.25)
            albedo = go_tail(albedo)
            albedo = go_tail(albedo)
            albedo = d_resize(albedo, sketch.shape)
            boundary = sketch.astype(np.float32)
            boundary = cv2.GaussianBlur(boundary, (0, 0), 1.618) - boundary
            boundary = boundary.clip(0, 255)
            albedo = cv2.cvtColor(albedo, cv2.COLOR_RGB2HSV).astype(np.float32)
            albedo[:, :, 1] += albedo[:, :, 1] * boundary / 48.0
            albedo[:, :, 2] -= boundary
            albedo = cv2.cvtColor(albedo.clip(0, 255).astype(np.uint8), cv2.COLOR_HSV2RGB)
        npcache(history, room, 'albedo.' + ID + '.png', albedo)
        print('albedo saved')
        if need_render == 0:
            npcache(history, room, 'result.' + ID + '.jpg', albedo)
            # cv2.imwrite('results/' + room + '.' + ID + '.jpg', albedo)
            print(time.time() - timenow)
            return room + '_' + ID, history
        HSV, YUV, DEL = process_albedo(albedo, composition, sketch)
        npcache(history, room, 'HSV.' + ID + '.jpg', HSV)
        npcache(history, room, 'YUV.' + ID + '.jpg', YUV)
        npcache(history, room, 'DEL.' + ID + '.jpg', DEL)
        print('HSV YUV DEL')
    albedo_s1024 = d_resize(albedo, s1024.shape)
    matting = go_mat(albedo_s1024)
    matting = np.tile(matting[:, :, None], [1, 1, 3])
    matting = shade_fill(matting, fill, opreate_normal_hint(ini_hint(s1024), points, length=2, skip_sp=False), s1024)
    matting = matting[:, :, 0]
    depth = np.zeros_like(matting, dtype=np.uint8) + 255
    depth[matting < 127] = 127
    depth[s1024 < 250] = 0
    npcache(history, room, 'depth.' + ID + '.jpg', depth)
    print('depth saved')
    normal = go_norm(depth).astype(np.float32)
    normal = ((normal + 1e-4) / (np.max(normal, axis=2, keepdims=True) + 1e-4) * 255.0).clip(0, 255).astype(np.uint8)
    normal[matting < 127] = 255
    normal = re_deatlize(normal, s1024)
    normal = d_resize(normal, sketch.shape)
    npcache(history, room, 'normal.' + ID + '.jpg', normal)
    print('norm saved')
    mask = np.zeros_like(matting, dtype=np.uint8) + 255
    mask[matting < 127] = 0
    mask = d_resize(mask, sketch.shape)
    mask[mask < 127] = 0
    mask[mask > 0] = 255
    if int(light_d) == 0:
        result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=True, top=True)
    elif int(light_d) == 1:
        result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=False, top=True)
    elif int(light_d) == 2:
        result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=True, top=False)
    else:
        result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=False, top=False)
    if need_render == 2:
        npcache(history, room, 'result.' + ID + '.jpg', result)
        # cv2.imwrite('results/' + room + '.' + ID + '.jpg', result)
        print(time.time() - timenow)
        return room + '_' + ID, history
    print('result saved')
    preview = np.concatenate([np.tile(sketch[:, :, None], [1, 1, 3]), albedo, result], axis=1)
    npcache(history, room, 'preview.' + ID + '.jpg', preview)
    print('preview saved')
    npcache(history, room, 'result.' + ID + '.jpg', result)
    # cv2.imwrite('results/' + room + '.' + ID + '.jpg', preview)
    print(time.time() - timenow)
    return room + '_' + ID, history


def download_result(name, history):
    room_id, name = name.split('/')
    rooms = list(filter(lambda _room: _room["id"] == room_id, history))
    if len(rooms) == 0:
        return None
    else:
        room = rooms[0]

    real_name = None
    for k in room.keys():
        if name in k:
            real_name = k
            break
    if real_name is None:
        return None
    name = real_name
    
    result = room.get(name, None)

    if 'points' in name:
        return json.dumps(result)
    
    return cv2_encode(result, name)
    

with gr.Blocks() as demo:
    history = gr.State(value=[])
    with gr.Row():
        with gr.Column():
            btn_show = gr.Button("Open Style2Paints V4.2")
    btn_show.click(None, _js="(_) => open('file/ui/web-mobile/index.html')")

    with gr.Row():
        with gr.Box():
            with gr.Row():
                upload_sketch_json = gr.Textbox(label="upload_sketch(json string)")
            with gr.Row():
                upload_sketch_btn = gr.Button(label="Submit sketch json")
            with gr.Row():
                upload_sketch_result = gr.Textbox(label="Result", interactive=False)
            upload_sketch_btn.click(upload_sketch, [upload_sketch_json, history], [upload_sketch_result, history], api_name="upload_sketch")
        
        with gr.Box():
            with gr.Row():
                request_result_json = gr.Textbox(label="request_result(json string)")
            with gr.Row():
                request_result_btn = gr.Button(label="Submit json of request for result")
            with gr.Row():
                request_result_result = gr.Textbox(label="Result", interactive=False)
            upload_sketch_btn.click(request_result, [request_result_json, history], [request_result_result, history], api_name="request_result")
        
        with gr.Box():
            with gr.Row():
                download_result_json = gr.Textbox(label="download_result(json string)")
            with gr.Row():
                download_result_btn = gr.Button(label="Submit json of download for result")
            with gr.Row():
                download_result_result = gr.Textbox(label="Result", interactive=False)
            upload_sketch_btn.click(download_result, [download_result_json, history], [download_result_result], api_name="download_result")

demo.launch()