init app

InstanceNorm.py

@@ -0,0 +1,146 @@
+from keras.engine import Layer, InputSpec
+from keras import initializers, regularizers, constraints
+from keras import backend as K
+from keras.utils.generic_utils import get_custom_objects
+
+import tensorflow as tf
+
+
+class InstanceNormalization(Layer):
+    """Instance normalization layer (Lei Ba et al, 2016, Ulyanov et al., 2016).
+    Normalize the activations of the previous layer at each step,
+    i.e. applies a transformation that maintains the mean activation
+    close to 0 and the activation standard deviation close to 1.
+    # Arguments
+        axis: Integer, the axis that should be normalized
+            (typically the features axis).
+            For instance, after a `Conv2D` layer with
+            `data_format="channels_first"`,
+            set `axis=1` in `InstanceNormalization`.
+            Setting `axis=None` will normalize all values in each instance of the batch.
+            Axis 0 is the batch dimension. `axis` cannot be set to 0 to avoid errors.
+        epsilon: Small float added to variance to avoid dividing by zero.
+        center: If True, add offset of `beta` to normalized tensor.
+            If False, `beta` is ignored.
+        scale: If True, multiply by `gamma`.
+            If False, `gamma` is not used.
+            When the next layer is linear (also e.g. `nn.relu`),
+            this can be disabled since the scaling
+            will be done by the next layer.
+        beta_initializer: Initializer for the beta weight.
+        gamma_initializer: Initializer for the gamma weight.
+        beta_regularizer: Optional regularizer for the beta weight.
+        gamma_regularizer: Optional regularizer for the gamma weight.
+        beta_constraint: Optional constraint for the beta weight.
+        gamma_constraint: Optional constraint for the gamma weight.
+    # Input shape
+        Arbitrary. Use the keyword argument `input_shape`
+        (tuple of integers, does not include the samples axis)
+        when using this layer as the first layer in a model.
+    # Output shape
+        Same shape as input.
+    # References
+        - [Layer Normalization](https://arxiv.org/abs/1607.06450)
+        - [Instance Normalization: The Missing Ingredient for Fast Stylization](https://arxiv.org/abs/1607.08022)
+    """
+    def __init__(self,
+                 axis=None,
+                 epsilon=1e-3,
+                 center=True,
+                 scale=True,
+                 beta_initializer='zeros',
+                 gamma_initializer='ones',
+                 beta_regularizer=None,
+                 gamma_regularizer=None,
+                 beta_constraint=None,
+                 gamma_constraint=None,
+                 **kwargs):
+        super(InstanceNormalization, self).__init__(**kwargs)
+        self.supports_masking = True
+        self.axis = axis
+        self.epsilon = epsilon
+        self.center = center
+        self.scale = scale
+        self.beta_initializer = initializers.get(beta_initializer)
+        self.gamma_initializer = initializers.get(gamma_initializer)
+        self.beta_regularizer = regularizers.get(beta_regularizer)
+        self.gamma_regularizer = regularizers.get(gamma_regularizer)
+        self.beta_constraint = constraints.get(beta_constraint)
+        self.gamma_constraint = constraints.get(gamma_constraint)
+
+    def build(self, input_shape):
+        ndim = len(input_shape)
+        if self.axis == 0:
+            raise ValueError('Axis cannot be zero')
+
+        if (self.axis is not None) and (ndim == 2):
+            raise ValueError('Cannot specify axis for rank 1 tensor')
+
+        self.input_spec = InputSpec(ndim=ndim)
+
+        if self.axis is None:
+            shape = (1,)
+        else:
+            shape = (input_shape[self.axis],)
+
+        if self.scale:
+            self.gamma = self.add_weight(shape=shape,
+                                         name='gamma',
+                                         initializer=self.gamma_initializer,
+                                         regularizer=self.gamma_regularizer,
+                                         constraint=self.gamma_constraint)
+        else:
+            self.gamma = None
+        if self.center:
+            self.beta = self.add_weight(shape=shape,
+                                        name='beta',
+                                        initializer=self.beta_initializer,
+                                        regularizer=self.beta_regularizer,
+                                        constraint=self.beta_constraint)
+        else:
+            self.beta = None
+        self.built = True
+
+    def call(self, inputs, training=None):
+        input_shape = K.int_shape(inputs)
+        reduction_axes = list(range(0, len(input_shape)))
+
+        if (self.axis is not None):
+            del reduction_axes[self.axis]
+
+        del reduction_axes[0]
+
+        mean, var = tf.nn.moments(inputs, reduction_axes, keep_dims=True)
+        stddev = tf.sqrt(var) + self.epsilon
+        normed = (inputs - mean) / stddev
+
+        broadcast_shape = [1] * len(input_shape)
+        if self.axis is not None:
+            broadcast_shape[self.axis] = input_shape[self.axis]
+
+        if self.scale:
+            broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
+            normed = normed * broadcast_gamma
+        if self.center:
+            broadcast_beta = K.reshape(self.beta, broadcast_shape)
+            normed = normed + broadcast_beta
+        return normed
+
+    def get_config(self):
+        config = {
+            'axis': self.axis,
+            'epsilon': self.epsilon,
+            'center': self.center,
+            'scale': self.scale,
+            'beta_initializer': initializers.serialize(self.beta_initializer),
+            'gamma_initializer': initializers.serialize(self.gamma_initializer),
+            'beta_regularizer': regularizers.serialize(self.beta_regularizer),
+            'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
+            'beta_constraint': constraints.serialize(self.beta_constraint),
+            'gamma_constraint': constraints.serialize(self.gamma_constraint)
+        }
+        base_config = super(InstanceNormalization, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+get_custom_objects().update({'InstanceNormalization': InstanceNormalization})

ai.py

@@ -0,0 +1,203 @@
+import tensorflow
+
+tensorflow.compat.v1.disable_v2_behavior()
+tf = tensorflow.compat.v1
+
+import keras
+import numpy as np
+from config import *
+from keras.models import load_model
+from smoother import *
+import keras.backend as K
+from models import *
+
+
+def ToGray(x):
+    R = x[:, :, :, 0:1]
+    G = x[:, :, :, 1:2]
+    B = x[:, :, :, 2:3]
+    return 0.30 * R + 0.59 * G + 0.11 * B
+
+
+def RGB2YUV(x):
+    R = x[:, :, :, 0:1]
+    G = x[:, :, :, 1:2]
+    B = x[:, :, :, 2:3]
+    Y = 0.299 * R + 0.587 * G + 0.114 * B
+    U = 0.492 * (B - Y) + 128
+    V = 0.877 * (R - Y) + 128
+    return tf.concat([Y, U, V], axis=3)
+
+
+def YUV2RGB(x):
+    Y = x[:, :, :, 0:1]
+    U = x[:, :, :, 1:2]
+    V = x[:, :, :, 2:3]
+    R = Y + 1.140 * (V - 128)
+    G = Y - 0.394 * (U - 128) - 0.581 * (V - 128)
+    B = Y + 2.032 * (U - 128)
+    return tf.concat([R, G, B], axis=3)
+
+
+def VGG2RGB(x):
+    return (x + [103.939, 116.779, 123.68])[:, :, :, ::-1]
+
+
+def blur(x):
+    return Smoother({'data': tf.pad(x, [[0, 0], [9, 9], [9, 9], [0, 0]], 'SYMMETRIC')}, 7, 2).get_output()[:, 9: -9, 9: -9, :]
+
+
+def norm_feature(x, core):
+    cs0 = tf.shape(core)[1]
+    cs1 = tf.shape(core)[2]
+    small = tf.image.resize_area(x, (cs0, cs1))
+    avged = tf.nn.avg_pool(tf.pad(small, [[0, 0], [2, 2], [2, 2], [0, 0]], 'REFLECT'), [1, 5, 5, 1], [1, 1, 1, 1], 'VALID')
+    return tf.image.resize_bicubic(avged, tf.shape(x)[1:3])
+
+
+def upsample(x):
+    return K.resize_images(x, 2, 2, 'channels_last')
+
+
+def downsample(x):
+    return tf.nn.avg_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
+
+
+def nts(x):
+    return (x + [103.939, 116.779, 123.68])[:, :, :, ::-1] / 255.0
+
+
+session = keras.backend.get_session()
+
+ip1 = tf.placeholder(dtype=tf.float32, shape=(None, None, None, 1))
+ip3 = tf.placeholder(dtype=tf.float32, shape=(None, None, None, 3))
+ip4 = tf.placeholder(dtype=tf.float32, shape=(None, None, None, 4))
+
+print('1')
+
+vector = make_diff_net()
+vector_op = 255.0 - tf.nn.sigmoid(vector(ip3 / 255.0)) * 255.0
+
+print('4')
+
+reader = load_model('./nets/reader.net')
+features = reader(ip3 / 255.0)
+
+print('5')
+
+head = load_model('./nets/head.net')
+feed = [1 - ip1 / 255.0, (ip4[:, :, :, 0:3] / 127.5 - 1) * ip4[:, :, :, 3:4] / 255.0]
+for _ in range(len(features)):
+    feed.append(keras.backend.mean(features[_], axis=[1, 2]))
+nil0, nil1, head_temp = head(feed)
+
+print('6')
+
+neck = load_model('./nets/neck.net')
+nil2, nil3, neck_temp = neck(feed)
+feed[0] = tf.clip_by_value(1 - tf.image.resize_bilinear(ToGray(VGG2RGB(head_temp) / 255.0), tf.shape(ip1)[1:3]), 0.0, 1.0)
+nil4, nil5, head_temp = neck(feed)
+head_op = VGG2RGB(head_temp)
+neck_op = VGG2RGB(neck_temp)
+
+print('7')
+
+inception = load_model('./nets/inception.net')
+features_render = inception((ip3 + (downsample(ip1) - blur(downsample(ip1))) * 2.0) / 255.0)
+precessed_feed = [(ip4[:, :, :, 0:3] / 127.5 - 1) * ip4[:, :, :, 3:4] / 255.0] + [
+    norm_feature(item, features_render[-1]) for item in features_render]
+
+print('8')
+
+render_head = load_model('./nets/render_head.net')
+render_neck = load_model('./nets/render_neck.net')
+nil6, nil7, render_A = render_head([1 - ip1 / 255.0] + precessed_feed)
+nil8, nil9, render_B = render_neck(
+    [1 - tf.image.resize_bilinear(ToGray(nts(render_A)), tf.shape(ip1)[1:3])] + precessed_feed)
+render_op = nts(render_B) * 255.0
+
+print('9')
+
+tail = load_model('./nets/tail.net')
+pads = 7
+tail_op = tail(tf.pad(ip3 / 255.0, [[0, 0], [pads, pads], [pads, pads], [0, 0]], 'REFLECT'))[:, pads * 2:-pads * 2, pads * 2:-pads * 2, :][:, 1:-1, 1:-1, :] * 255.0
+
+print('10')
+
+
+vgg7 = load_model('./nets/vgg7.net')
+pads = 7
+vgg7_op = vgg7(tf.pad(ip1 / 255.0, [[0, 0], [pads, pads], [pads, pads], [0, 0]], 'REFLECT'))[:, pads:-pads, pads:-pads, :] * 255.0
+
+print('11')
+
+
+mat = make_unet512()
+mat_op = mat(ip3 / 255.0) * 255.0
+
+print('11')
+
+norm = load_model('./nets/norm.net')
+norm_op = norm(ip1 / 255.0) * 255.0
+
+print('12')
+
+session.run(tf.global_variables_initializer())
+
+print('begin load')
+
+
+tail.load_weights('./nets/tail.net')
+vgg7.load_weights('./nets/vgg7.net')
+head.load_weights('./nets/head.net')
+neck.load_weights('./nets/neck.net')
+reader.load_weights('./nets/reader.net')
+vector.load_weights('./nets/vector.net')
+render_head.load_weights('./nets/render_head.net')
+render_neck.load_weights('./nets/render_neck.net')
+inception.load_weights('./nets/inception.net')
+mat.load_weights('./nets/mat.net')
+norm.load_weights('./nets/norm.net')
+
+
+def go_head(sketch, global_hint, local_hint):
+    return session.run(head_op, feed_dict={
+        ip1: sketch[None, :, :, None], ip3: global_hint[None, :, :, :], ip4: local_hint[None, :, :, :]
+    })[0].clip(0, 255).astype(np.uint8)
+
+
+def go_render(sketch, segmentation, points):
+    return session.run(render_op, feed_dict={
+        ip1: sketch[None, :, :, None], ip3: segmentation[None, :, :, :], ip4: points[None, :, :, :]
+    })[0].clip(0, 255).astype(np.uint8)
+
+
+def go_tail(x):
+    return session.run(tail_op, feed_dict={
+        ip3: x[None, :, :, :]
+    })[0].clip(0, 255).astype(np.uint8)
+
+
+def go_vgg7(x):
+    return session.run(vgg7_op, feed_dict={
+        ip1: x[None, :, :, None]
+    })[0, :, :, 0].clip(0, 255).astype(np.uint8)
+
+
+def go_vector(x):
+    return session.run(vector_op, feed_dict={
+        ip3: x[None, :, :, :]
+    })[0].clip(0, 255).astype(np.uint8)
+
+
+def go_mat(x):
+    return session.run(mat_op, feed_dict={
+        ip3: x[None, :, :, :]
+    })[0, :, :, 0].clip(0, 255).astype(np.uint8)
+
+
+def go_norm(x):
+    return session.run(norm_op, feed_dict={
+        ip1: x[None, :, :, None]
+    })[0].clip(0, 255).astype(np.uint8)
+

app.py

@@ -0,0 +1,304 @@
+import re
+import base64
+import datetime
+import sys
+import pickle
+import gzip
+import json
+import time
+import gradio as gr
+
+from ai import *
+from tricks import *
+from decompositioner import *
+from rendering import *
+
+
+import os
+import glob
+import shutil
+
+splash = glob.glob('ui/web-mobile/splash*')[0]
+os.remove(splash)
+shutil.copy('res/splash.png', splash)
+with open('ui/web-mobile/index.html', 'r', encoding='utf-8') as f:
+    page = f.read()
+with open('ui/web-mobile/index.html', 'w', encoding='utf-8') as f:
+    f.write(page.replace('Cocos Creator | ', ''))
+
+
+def cv2_encode(image: np.ndarray, name):
+    if image is None:
+        return 'null'
+    if '.jpg' in name:
+        _, data = cv2.imencode('.jpeg', image)
+        return 'data:image/jpeg;base64,' + base64.b64encode(data).decode('utf8')
+    else:
+        _, data = cv2.imencode('.png', image)
+        return 'data:image/png;base64,' + base64.b64encode(data).decode('utf8')
+
+
+def get_request_image(request, name):
+    img = request.get(name)
+    img = re.sub('^data:image/.+;base64,', '', img)
+    img = base64.b64decode(img)
+    img = np.fromstring(img, dtype=np.uint8)
+    img = cv2.imdecode(img, -1)
+    return img
+
+
+def npcache(history, room_id, name, data):
+    rooms = list(filter(lambda _room: _room["id"] == room_id, history))
+    if len(rooms) == 0:
+        room = { "id": room_id }
+        history.append(room)
+    else:
+        room = rooms[0]
+    room[name] = data
+
+
+def npread(history, room_id, name):
+    rooms = list(filter(lambda _room: _room["id"] == room_id, history))
+    if len(rooms) == 0:
+        return None
+    else:
+        room = rooms[0]
+    return room.get(name, None)
+
+
+def upload_sketch(json_str, history):
+    request = json.loads(json_str)
+    timenow = time.time()
+    ID = datetime.datetime.now().strftime('H%HM%MS%S')
+    room = datetime.datetime.now().strftime('%b%dH%HM%MS%S') + 'R' + str(np.random.randint(100, 999))
+    sketch = from_png_to_jpg(get_request_image(request, 'sketch'))
+    npcache(history, room, 'sketch.original.jpg', sketch)
+    sketch = go_tail(cli_norm(min_resize(sketch, 512)))
+    print('original_sketch saved')
+    s256 = go_vector(go_cal(mk_resize(sketch, 8)))[:, :, 0]
+    print('s256')
+    s512 = go_vector(go_cal(d_resize(sketch, s256.shape, 2.0)))[:, :, 0]
+    print('s512')
+    s1024 = go_vector(go_cal(d_resize(sketch, s256.shape, 4.0)))[:, :, 0]
+    print('s1024')
+    npcache(history, room, 'sketch.s1024.png', s1024)
+    npcache(history, room, 'sketch.s512.png', s512)
+    npcache(history, room, 'sketch.s256.png', s256)
+    print('edge processed')
+    fill = double_fill(s1024, s512, s256)
+    npcache(history, room, 'sketch.fill', fill)
+    print('filled')
+    npcache(history, room, 'sketch.colorization.png', np.min(sketch, axis=2))
+    print('sketch processed')
+    print(time.time() - timenow)
+    if len(history) > 5:
+        history = history[-5:]
+    return room + '_' + ID, history
+
+
+def request_result(json_str, history):
+    request = json.loads(json_str)
+    timenow = time.time()
+    room = request.get("room")
+    if len(list(filter(lambda _room: _room["id"] == room, history))) == 0:
+        return None, history
+    skipper = str(request.get("skipper"))
+    light_r = float(request.get("r"))
+    light_g = float(request.get("g"))
+    light_b = float(request.get("b"))
+    light_h = float(request.get("h"))
+    light_max = max([light_r, light_g, light_b, light_h])
+    inv4 = int(request.get("inv4"))
+    print('inv4=' + str(inv4))
+    light_r = (light_r + 1e-5) / (light_max + 1e-5)
+    light_g = (light_g + 1e-5) / (light_max + 1e-5)
+    light_b = (light_b + 1e-5) / (light_max + 1e-5)
+    light_h *= 600.0
+    light_d = request.get("d")
+    need_render = int(request.get("need_render"))
+    print([light_r, light_g, light_b, light_d])
+    ID = datetime.datetime.now().strftime('H%HM%MS%S')
+    points = request.get("points")
+    points = json.loads(points)
+    npcache(history, room, 'points.' + ID + '.txt', points)
+    if len(points) > 500:
+        return None, history
+    for _ in range(len(points)):
+        points[_][1] = 1 - points[_][1]
+    fill = npread(history, room, 'sketch.fill')
+    s1024 = npread(history, room, 'sketch.s1024.png')
+    sketch = npread(history, room, 'sketch.colorization.png')
+    print(skipper)
+    if npread(history, room, 'albedo.' + skipper + '.png') is not None:
+        albedo = npread(history, room, 'albedo.' + skipper + '.png')
+        npcache(history, room, 'albedo.' + ID + '.png', albedo)
+        print('albedo readed')
+    else:
+        faceID = int(request.get("faceID")) - 65535
+        print(faceID)
+        if faceID > -1:
+            print('fake ref')
+            face = from_png_to_jpg(cv2.imread("refs/" + str(faceID + 1) + ".png", cv2.IMREAD_UNCHANGED))
+        else:
+            print('get ref')
+            face = from_png_to_jpg(get_request_image(request, 'face'))
+        npcache(history, room, 'face.' + ID + '.jpg', face)
+        face = s_enhance(face)
+        print('request result room = ' + str(room) + ', ID = ' + str(ID))
+        print('processing painting in ' + room)
+        if inv4 > 0:
+            sketch_1024 = k_resize(sketch, 64)
+        else:
+            sketch_1024 = k_resize(sketch, 48)
+        hints_1024 = ini_hint(sketch_1024)
+        hints_1024 = opreate_normal_hint(hints_1024, points, length=2, skip_sp=True)
+        baby = go_head(
+            sketch=sketch_1024,
+            global_hint=k_resize(face, 14),
+            local_hint=hints_1024
+        )
+        npcache(history, room, 'baby.' + ID + '.jpg', baby)
+        print('baby born')
+        composition = d_resize(re_deatlize(deatlize(balance_fill(baby, fill, opreate_normal_hint(ini_hint(s1024), points, length=2, skip_sp=True), s1024)), s1024), sketch.shape)
+        npcache(history, room, 'composition.' + ID + '.jpg', composition)
+        gird = process_overlay(composition, sketch)
+        npcache(history, room, 'gird.' + ID + '.jpg', gird)
+        print('composition saved')
+        if inv4 > 0:
+            albedo = go_render(sketch_1024, d_resize(composition, sketch_1024.shape, 0.5), hints_1024)
+            albedo = go_tail(albedo)
+            albedo = d_resize(re_deatlize(d_resize(albedo, s1024.shape), s1024), sketch.shape)
+            albedo = cv2.cvtColor(albedo, cv2.COLOR_RGB2YUV)
+            albedo[:, :, 0] = go_vgg7(albedo[:, :, 0])
+            albedo = cv2.cvtColor(albedo, cv2.COLOR_YUV2RGB)
+        else:
+            albedo = re_deatlize(d_resize(baby, s1024.shape), s1024)
+            albedo = d_resize(albedo, sketch.shape, 0.25)
+            albedo = go_tail(albedo)
+            albedo = go_tail(albedo)
+            albedo = d_resize(albedo, sketch.shape)
+            boundary = sketch.astype(np.float32)
+            boundary = cv2.GaussianBlur(boundary, (0, 0), 1.618) - boundary
+            boundary = boundary.clip(0, 255)
+            albedo = cv2.cvtColor(albedo, cv2.COLOR_RGB2HSV).astype(np.float32)
+            albedo[:, :, 1] += albedo[:, :, 1] * boundary / 48.0
+            albedo[:, :, 2] -= boundary
+            albedo = cv2.cvtColor(albedo.clip(0, 255).astype(np.uint8), cv2.COLOR_HSV2RGB)
+        npcache(history, room, 'albedo.' + ID + '.png', albedo)
+        print('albedo saved')
+        if need_render == 0:
+            npcache(history, room, 'result.' + ID + '.jpg', albedo)
+            # cv2.imwrite('results/' + room + '.' + ID + '.jpg', albedo)
+            print(time.time() - timenow)
+            return room + '_' + ID, history
+        HSV, YUV, DEL = process_albedo(albedo, composition, sketch)
+        npcache(history, room, 'HSV.' + ID + '.jpg', HSV)
+        npcache(history, room, 'YUV.' + ID + '.jpg', YUV)
+        npcache(history, room, 'DEL.' + ID + '.jpg', DEL)
+        print('HSV YUV DEL')
+    albedo_s1024 = d_resize(albedo, s1024.shape)
+    matting = go_mat(albedo_s1024)
+    matting = np.tile(matting[:, :, None], [1, 1, 3])
+    matting = shade_fill(matting, fill, opreate_normal_hint(ini_hint(s1024), points, length=2, skip_sp=False), s1024)
+    matting = matting[:, :, 0]
+    depth = np.zeros_like(matting, dtype=np.uint8) + 255
+    depth[matting < 127] = 127
+    depth[s1024 < 250] = 0
+    npcache(history, room, 'depth.' + ID + '.jpg', depth)
+    print('depth saved')
+    normal = go_norm(depth).astype(np.float32)
+    normal = ((normal + 1e-4) / (np.max(normal, axis=2, keepdims=True) + 1e-4) * 255.0).clip(0, 255).astype(np.uint8)
+    normal[matting < 127] = 255
+    normal = re_deatlize(normal, s1024)
+    normal = d_resize(normal, sketch.shape)
+    npcache(history, room, 'normal.' + ID + '.jpg', normal)
+    print('norm saved')
+    mask = np.zeros_like(matting, dtype=np.uint8) + 255
+    mask[matting < 127] = 0
+    mask = d_resize(mask, sketch.shape)
+    mask[mask < 127] = 0
+    mask[mask > 0] = 255
+    if int(light_d) == 0:
+        result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=True, top=True)
+    elif int(light_d) == 1:
+        result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=False, top=True)
+    elif int(light_d) == 2:
+        result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=True, top=False)
+    else:
+        result = small_render(normal, mask, albedo, s1024, r=light_r, g=light_g, b=light_b, h=light_h, left=False, top=False)
+    if need_render == 2:
+        npcache(history, room, 'result.' + ID + '.jpg', result)
+        # cv2.imwrite('results/' + room + '.' + ID + '.jpg', result)
+        print(time.time() - timenow)
+        return room + '_' + ID, history
+    print('result saved')
+    preview = np.concatenate([np.tile(sketch[:, :, None], [1, 1, 3]), albedo, result], axis=1)
+    npcache(history, room, 'preview.' + ID + '.jpg', preview)
+    print('preview saved')
+    npcache(history, room, 'result.' + ID + '.jpg', result)
+    # cv2.imwrite('results/' + room + '.' + ID + '.jpg', preview)
+    print(time.time() - timenow)
+    return room + '_' + ID, history
+
+
+def download_result(name, history):
+    room_id, name = name.split('/')
+    rooms = list(filter(lambda _room: _room["id"] == room_id, history))
+    if len(rooms) == 0:
+        return None
+    else:
+        room = rooms[0]
+
+    real_name = None
+    for k in room.keys():
+        if name in k:
+            real_name = k
+            break
+    if real_name is None:
+        return None
+    name = real_name
+    
+    result = room.get(name, None)
+
+    if 'points' in name:
+        return json.dumps(result)
+    
+    return cv2_encode(result, name)
+    
+
+with gr.Blocks() as demo:
+    history = gr.State(value=[])
+    with gr.Row():
+        with gr.Column():
+            btn_show = gr.Button("Open Style2Paints V4.2")
+    btn_show.click(None, _js="(_) => open('file/ui/web-mobile/index.html')")
+
+    with gr.Row():
+        with gr.Box():
+            with gr.Row():
+                upload_sketch_json = gr.Textbox(label="upload_sketch(json string)")
+            with gr.Row():
+                upload_sketch_btn = gr.Button(label="Submit sketch json")
+            with gr.Row():
+                upload_sketch_result = gr.Textbox(label="Result", interactive=False)
+            upload_sketch_btn.click(upload_sketch, [upload_sketch_json, history], [upload_sketch_result, history], api_name="upload_sketch")
+        
+        with gr.Box():
+            with gr.Row():
+                request_result_json = gr.Textbox(label="request_result(json string)")
+            with gr.Row():
+                request_result_btn = gr.Button(label="Submit json of request for result")
+            with gr.Row():
+                request_result_result = gr.Textbox(label="Result", interactive=False)
+            upload_sketch_btn.click(request_result, [request_result_json, history], [request_result_result, history], api_name="request_result")
+        
+        with gr.Box():
+            with gr.Row():
+                download_result_json = gr.Textbox(label="download_result(json string)")
+            with gr.Row():
+                download_result_btn = gr.Button(label="Submit json of download for result")
+            with gr.Row():
+                download_result_result = gr.Textbox(label="Result", interactive=False)
+            upload_sketch_btn.click(download_result, [download_result_json, history], [download_result_result], api_name="download_result")
+
+demo.launch()

config.py

@@ -0,0 +1,2 @@
+multiple_process = True
+

decompositioner.py

@@ -0,0 +1,185 @@
+import os
+import numpy as np
+from scipy.spatial import ConvexHull
+from sklearn.cluster import MiniBatchKMeans
+from tricks import *
+import cv2
+
+
+ksd = 8
+mbc = MiniBatchKMeans(ksd)
+
+
+def get_theme(img):
+    images = np.reshape(cv2.resize(img, (256, 256)), (256 * 256, 3))
+    hull = ConvexHull(images)
+    return hull.points[hull.vertices]
+
+
+def simplify_points(points, img):
+    labels = mbc.fit(points)
+    new_points = []
+    all_center = np.mean(labels.cluster_centers_, axis=0)
+    distances = np.sum((points - all_center) ** 2, axis=1) ** 0.5
+
+    for idx in range(ksd):
+        candidates = points[labels.labels_ == idx]
+        scores = distances[labels.labels_ == idx]
+        best_id = np.argmax(scores)
+        new_points.append(candidates[best_id])
+
+    new_points.sort(key=np.sum, reverse=True)
+
+    new_points = np.stack(new_points, axis=0)
+    return new_points.clip(0, 255).astype(np.uint8)
+
+
+def get_ini_layers(miku, points):
+    results = []
+    final_target = miku.astype(np.float32)
+    bg = np.zeros_like(final_target, dtype=np.float32) + points[0]
+    results.append(np.concatenate([bg, np.zeros_like(bg, dtype=np.float32) + 255], axis=2)[:, :, 0:4])
+    current_result = bg.copy()
+    for layer_index in range(1, ksd):
+        current_base = current_result.astype(np.float32)
+        current_color = np.zeros_like(final_target, dtype=np.float32) + points[layer_index]
+        overall_direction = final_target - current_base
+        avaliable_direction = current_color - current_base
+        current_alpha = np.sum(overall_direction * avaliable_direction, axis=2, keepdims=True) / np.sum(
+            avaliable_direction * avaliable_direction, axis=2, keepdims=True)
+        current_alpha = current_alpha.clip(0, 1)
+        current_result = (current_color * current_alpha + current_base * (1 - current_alpha)).clip(0, 255)
+        results.append(np.concatenate([current_color, current_alpha * 255.0], axis=2))
+    return results
+
+
+def make_reconstruction(layers):
+    bg = np.zeros_like(layers[0], dtype=np.float32)[:, :, 0:3] + 255
+    for item in layers:
+        current_alpha = item[:, :, 3:4] / 255.0
+        bg = item[:, :, 0:3] * current_alpha + bg * (1 - current_alpha)
+    return bg
+
+
+def improve_layers(layers, miku):
+    reconstruction = make_reconstruction(layers)
+    b = miku - reconstruction
+    new_layers = []
+    for item in layers:
+        new_item = item.copy()
+        new_item[:, :, 0:3] = (new_item[:, :, 0:3] + b).clip(0, 255)
+        new_layers.append(new_item)
+    return new_layers
+
+
+def cluster_all(labeled_array, num_features):
+    xs = [[] for _ in range(num_features)]
+    ys = [[] for _ in range(num_features)]
+    M = labeled_array.shape[0]
+    N = labeled_array.shape[1]
+    for x in range(M):
+        for y in range(N):
+            i = labeled_array[x, y]
+            xs[i].append(x)
+            ys[i].append(y)
+    result = []
+    for _ in range(num_features):
+        result.append((np.array(xs[_]), np.array(ys[_])))
+    return result
+
+
+def meder(x):
+    y = x.copy()
+    y = cv2.medianBlur(y, 5)
+    y = cv2.medianBlur(y, 5)
+    y = cv2.medianBlur(y, 3)
+    y = cv2.medianBlur(y, 3)
+    return y
+
+
+def re_med(s_2048):
+
+    sample_2048 = s_2048.astype(np.float32)
+    sample_1024 = cv2.pyrDown(sample_2048)
+    sample_512 = cv2.pyrDown(sample_1024)
+    sample_256 = cv2.pyrDown(sample_512)
+
+    gradient_2048 = sample_2048 - cv2.pyrUp(sample_1024)
+    gradient_1024 = sample_1024 - cv2.pyrUp(sample_512)
+    gradient_512 = sample_512 - cv2.pyrUp(sample_256)
+
+    rec_256 = meder(sample_256)
+    rec_512 = cv2.pyrUp(rec_256) + meder(gradient_512)
+    rec_1024 = cv2.pyrUp(rec_512) + meder(gradient_1024)
+    rec_2048 = cv2.pyrUp(rec_1024) + meder(gradient_2048)
+    return rec_2048
+
+
+def process_ctx(sketch, solid, render):
+    solid = solid.astype(np.float32)
+    sketch = d_resize(cv2.cvtColor(sketch, cv2.COLOR_GRAY2RGB), solid.shape).astype(np.float32)
+    render = d_resize(render, solid.shape).astype(np.float32)
+    alpha = sketch / 255.0
+    all_diff = render - solid
+    all_lines = render.copy()
+    all_lines = cv2.erode(all_lines, np.ones((3,3), np.uint8)) * 0.618
+    all_diff = re_med(all_diff)
+    all_lines = re_med(all_lines)
+    recon = solid + all_diff
+    recon = recon * alpha + all_lines * (1 - alpha)
+    recon2 = (solid + all_diff) * alpha + re_med(solid) * (1 - alpha)
+    recon3 = reason_blending(recon2, sketch)
+    return recon.clip(0, 255).astype(np.uint8), recon2.clip(0, 255).astype(np.uint8), recon3.clip(0, 255).astype(np.uint8)
+
+
+def process_psd(sketch, solid, render, path='./'):
+    recon = process_ctx(sketch, solid, render)
+    points = get_theme(solid)
+    points = simplify_points(points, solid)
+    compositions = get_ini_layers(solid, points)
+    compositions = improve_layers(compositions, solid)
+    for _ in range(ksd):
+        cv2.imwrite(path + str(_ + 1) + '.color.png', compositions[_].clip(0, 255).astype(np.uint8))
+    solid = make_reconstruction(compositions).clip(0, 255).astype(np.uint8)
+    os.makedirs(path, exist_ok=True)
+    alpha = 1 - sketch.astype(np.float32) / 255.0
+    now = solid
+    now = (now.astype(np.float32) + sketch.astype(np.float32) - 255.0).clip(0, 255)
+    sketch = 255 + now - solid
+    cv2.imwrite(path + '9.sketch.png', sketch.clip(0, 255).astype(np.uint8))
+    all_diff = recon.astype(np.float32) - now
+    all_light = all_diff.copy()
+    all_shadow = - all_diff.copy()
+    all_light[all_light < 0] = 0
+    all_shadow[all_shadow < 0] = 0
+    sketch_color = all_light * alpha
+    light = all_light * (1 - alpha)
+    all_shadow = 255 - all_shadow
+    cv2.imwrite(path + '10.sketch_color.png', sketch_color.clip(0, 255).astype(np.uint8))
+    cv2.imwrite(path + '11.light.png', light.clip(0, 255).astype(np.uint8))
+    cv2.imwrite(path + '12.shadow.png', all_shadow.clip(0, 255).astype(np.uint8))
+    return recon
+
+
+def process_albedo(albedo, composition, sketch):
+    DEL = albedo.astype(np.float32)
+    HSV = cv2.cvtColor(albedo, cv2.COLOR_RGB2HSV).astype(np.float32)
+    YUV = cv2.cvtColor(albedo, cv2.COLOR_RGB2YUV).astype(np.float32)
+    solid = composition.astype(np.float32)
+    light = sketch[:, :, None].astype(np.float32)
+
+    DEL = DEL * light / 255.0 + solid * (1 - light / 255.0)
+    HSV[:, :, 2:3] = np.minimum(HSV[:, :, 2:3], light)
+    YUV[:, :, 0:1] = np.minimum(YUV[:, :, 0:1], light)
+
+    DEL = DEL.clip(0, 255).astype(np.uint8)
+    HSV = HSV.clip(0, 255).astype(np.uint8)
+    YUV = YUV.clip(0, 255).astype(np.uint8)
+
+    return cv2.cvtColor(HSV, cv2.COLOR_HSV2RGB), cv2.cvtColor(YUV, cv2.COLOR_YUV2RGB), DEL
+
+
+def process_overlay(composition, sketch):
+    RGB = composition.astype(np.float32)
+    alpha = sketch[:, :, None].astype(np.float32) / 255.0
+    return (RGB * alpha).clip(0, 255).astype(np.uint8)

generate_bash.py

@@ -0,0 +1,35 @@
+lines = []
+
+for _ in range(50, 66):
+    lines.append("screen -dmS \"server.80"+str(_)+"\" bash -c \'while : ;do python3 server.py 80"+str(_)+"; done;\'\n")
+
+with open('run.bash', 'wt') as f:
+    f.writelines(lines)
+
+lines = []
+
+for _ in range(50, 66):
+    lines.append("screen -S \"server.80"+str(_)+"\" -X quit\n")
+
+with open('kill.bash', 'wt') as f:
+    f.writelines(lines)
+
+lines = []
+
+for _ in range(50, 66):
+    lines.append("server 127.0.0.1:80"+str(_)+" weight=4 max_fails=2 fail_timeout=600s;\n")
+
+with open('nginx.txt', 'wt') as f:
+    f.writelines(lines)
+
+lines = ["ufw allow 80/tcp\n"]
+
+for _ in range(50, 66):
+    lines.append("ufw allow 80"+str(_)+"/tcp\n")
+
+with open('tcp.bash', 'wt') as f:
+    f.writelines(lines)
+
+#/etc/nginx/sites-available
+#/var/log/nginx
+

linefiller/thinning.py

@@ -0,0 +1,44 @@
+import numpy as np
+import cv2
+from numba import njit
+
+
+@njit
+def njit_thin(points, maps):
+    result = maps.copy()
+    h, w = maps.shape[:2]
+    for _ in range(len(points[0])):
+        x = points[0][_]
+        y = points[1][_]
+        if x > 0:
+            a = maps[x-1, y]
+            if a > 0:
+                result[x, y] = a
+                continue
+        if y > 0:
+            a = maps[x, y-1]
+            if a > 0:
+                result[x, y] = a
+                continue
+        if x + 1 < h:
+            a = maps[x+1, y]
+            if a > 0:
+                result[x, y] = a
+                continue
+        if y + 1 < w:
+            a = maps[x, y+1]
+            if a > 0:
+                result[x, y] = a
+                continue
+    return result
+
+
+def thinning(fillmap, max_iter=100):
+    result = fillmap.copy()
+    for iterNum in range(max_iter):
+        line_points = np.where(result == 0)
+        if not len(line_points[0]) > 0:
+            break
+        result = njit_thin(line_points, result)
+    return result
+

linefiller/third_party.py

@@ -0,0 +1,396 @@
+import cv2
+from .thinning import *
+from .trappedball_fill import *
+from skimage.measure import block_reduce
+from skimage.morphology import disk, dilation, erosion
+from numba import njit
+
+
+def np_min_pool(x):
+    return block_reduce(x, (2, 2), np.min)
+
+
+def np_max_pool(x):
+    return block_reduce(x, (2, 2), np.max)
+
+
+def np_max_441(x):
+    return block_reduce(x, (4, 4, 1), np.max)
+
+
+def np_max_pool_221(x):
+    return block_reduce(x, (2, 2, 1), np.max)
+
+
+def np_max_pool_s(x, s):
+    return block_reduce(x, (s, s, 1), np.max)
+
+
+def binarize(x):
+    xp = x.copy()
+    xp[xp < 250] = 0
+    xp[xp > 0] = 255
+    return xp
+
+
+def get_initial_fillmap(boundary, merge=True):
+    fillmap = build_fill_map(boundary, flood_fill_multi(boundary, merge=merge))
+    return fillmap
+
+
+def up_propagate(small_fillmap, big_boundary):
+    new_fillmap = cv2.resize(small_fillmap, (big_boundary.shape[1], big_boundary.shape[0]), interpolation=cv2.INTER_NEAREST)
+    padded_fillmap = np.pad(new_fillmap, [[1, 1], [1, 1]], 'constant', constant_values=0)
+    new_mask = np.ones_like(new_fillmap, dtype=np.uint8) * 255
+    new_mask[new_fillmap > 0] = 0
+    new_mask[big_boundary < 240] = 0
+    fills = flood_fill_multi(new_mask, merge=True)
+    max_id = np.max(new_fillmap)
+    for item in fills:
+        points0 = padded_fillmap[(item[0] + 1, item[1] + 0)]
+        points1 = padded_fillmap[(item[0] + 1, item[1] + 2)]
+        points2 = padded_fillmap[(item[0] + 0, item[1] + 1)]
+        points3 = padded_fillmap[(item[0] + 2, item[1] + 1)]
+
+        all_points = np.concatenate([points0, points1, points2, points3], axis=0)
+        pointsets, pointcounts = np.unique(all_points[all_points > 0], return_counts=True)
+
+        if len(pointsets) > 0:
+            new_fillmap[item] = pointsets[np.argmax(pointcounts)]
+        else:
+            max_id += 1
+            new_fillmap[item] = max_id
+    return new_fillmap
+
+
+def laplas_fill(b_512, b_256, b_128):
+    b_512 = binarize(b_512)
+    b_256 = binarize(b_256)
+    b_128 = binarize(b_128)
+    f128 = get_initial_fillmap(b_128)
+    f256 = up_propagate(f128, b_256)
+    f512 = up_propagate(f256, b_512)
+    fin = thinning(f512)
+    return fin
+
+
+@ njit
+def get_corner(x):
+    corner = x.copy()
+    s0 = corner.shape[0]
+    s1 = corner.shape[1]
+    for i0 in range(1, s0 - 1):
+        for i1 in range(1, s1 - 1):
+            if x[i0, i1] == 0:
+                continue
+            if x[i0, i1 - 1] == 0:
+                if x[i0 - 1, i1 - 1] == 0:
+                    continue
+                if x[i0 + 1, i1 - 1] == 0:
+                    continue
+                corner[i0, i1] = 0
+                continue
+            if x[i0, i1 + 1] == 0:
+                if x[i0 - 1, i1 + 1] == 0:
+                    continue
+                if x[i0 + 1, i1 + 1] == 0:
+                    continue
+                corner[i0, i1] = 0
+                continue
+            if x[i0 - 1, i1] == 0:
+                if x[i0 - 1, i1 - 1] == 0:
+                    continue
+                if x[i0 - 1, i1 + 1] == 0:
+                    continue
+                corner[i0, i1] = 0
+                continue
+            if x[i0 + 1, i1] == 0:
+                if x[i0 + 1, i1 - 1] == 0:
+                    continue
+                if x[i0 + 1, i1 + 1] == 0:
+                    continue
+                corner[i0, i1] = 0
+                continue
+    return corner
+
+
+def monogrouh(x):
+    y = 255 - x
+    y = dilation(y, disk(1))
+    y = dilation(y, disk(1))
+    y = erosion(y, disk(1))
+    y = erosion(y, disk(1))
+    y = 255 - y
+    return y
+
+
+def corners(x):
+    y = x.copy()
+    y = monogrouh(y)
+    y = get_corner(y)
+    y = monogrouh(y)
+    y = get_corner(y)
+    y = monogrouh(y)
+    return y
+
+
+def save_fill(name, fill):
+    cv2.imwrite(name, show_fill_map(fill))
+
+
+def double_fill(b_1024, b_512, b256):
+    b256 = binarize(b256)
+    b_512 = binarize(b_512)
+    b_1024 = binarize(b_1024)
+    b_1024 = corners(b_1024)
+    b_512 = np.min(np.stack([b_512, np_min_pool(b_1024)], axis=2), axis=2)
+    b_512 = corners(b_512)
+    b_256 = np.min(np.stack([b256, np_min_pool(b_512)], axis=2), axis=2)
+    b_256 = corners(b_256)
+    b_128 = np_min_pool(b_256)
+    b_128 = corners(b_128)
+    b_64 = np_min_pool(b_128)
+    f64 = get_initial_fillmap(b_64)
+    print('get_initial_fillmap(b_64)')
+    f128 = up_propagate(f64, b_128)
+    print('up_propagate(f64, b_128)')
+    f256 = up_propagate(f128, b_256)
+    print('up_propagate(f128, b_256)')
+    f512 = up_propagate(f256, b_512)
+    print('up_propagate(f256, b_512)')
+    f1024 = up_propagate(f512, b_1024)
+    print('up_propagate(f512, b_1024)')
+    fin = thinning(f1024)
+    print('thinning(f1024)')
+
+    # cv2.imwrite('b_64.png', b_64)
+    # cv2.imwrite('b_128.png', b_128)
+    # cv2.imwrite('b_256.png', b_256)
+    # cv2.imwrite('b_512.png', b_512)
+    # cv2.imwrite('b_1024.png', b_1024)
+    # save_fill('f64.png', f64)
+    # save_fill('f128.png', f128)
+    # save_fill('f256.png', f256)
+    # save_fill('f512.png', f512)
+    # save_fill('f1024.png', f1024)
+    # save_fill('fin.png', fin)
+
+    return find_all(fin)
+
+
+def single_fill(b_2048, path):
+    b_2048 = corners(binarize(b_2048))
+    f2048 = get_initial_fillmap(b_2048, merge=False)
+    print(path + 'get_initial_fillmap(b_2048, merge=False)')
+    fin = thinning(f2048)
+    print(path + 'thinning(f2048)')
+    # cv2.imwrite(path + 'b_2048.png', b_2048)
+    # save_fill(path + 'f2048.png', f2048)
+    # save_fill(path + 'fin.png', fin)
+    return find_all(fin)
+
+
+def deatlize(x):
+    x = cv2.GaussianBlur(x, (0, 0), 0.8)
+    x = cv2.medianBlur(x, 3)
+    return x
+
+
+def low_down(gradient_mask):
+    return 1.0 - cv2.dilate(255 - gradient_mask, np.ones((3, 3), np.uint8), iterations=2).astype(np.float32) / 255.0
+
+
+def cv2pyrDown(x):
+    return cv2.pyrDown(cv2.medianBlur(cv2.medianBlur(x, 3), 3))
+
+
+def cv2pyrUp(x):
+    return cv2.pyrUp(cv2.medianBlur(cv2.medianBlur(x, 3), 3))
+
+
+def re_deatlize(visulized, s1024):
+
+    gradient_mask_1024 = binarize(s1024)
+    gradient_mask_512 = np_min_pool(gradient_mask_1024)
+    gradient_mask_256 = np_min_pool(gradient_mask_512)
+    gradient_mask_128 = np_min_pool(gradient_mask_256)
+    gradient_mask_64 = np_min_pool(gradient_mask_128)
+
+    gradient_mask_1024 = low_down(gradient_mask_1024)
+    gradient_mask_512 = low_down(gradient_mask_512)
+    gradient_mask_256 = low_down(gradient_mask_256)
+    gradient_mask_128 = low_down(gradient_mask_128)
+    gradient_mask_64 = low_down(gradient_mask_64)
+
+    sample_1024 = visulized.astype(np.float32)
+    sample_512 = cv2pyrDown(sample_1024)
+    sample_256 = cv2pyrDown(sample_512)
+    sample_128 = cv2pyrDown(sample_256)
+    sample_64 = cv2pyrDown(sample_128)
+    sample_32 = cv2pyrDown(sample_64)
+
+    gradient_1024 = sample_1024 - cv2pyrUp(sample_512)
+    gradient_512 = sample_512 - cv2pyrUp(sample_256)
+    gradient_256 = sample_256 - cv2pyrUp(sample_128)
+    gradient_128 = sample_128 - cv2pyrUp(sample_64)
+    gradient_64 = sample_64 - cv2pyrUp(sample_32)
+
+    rec_32 = sample_32
+    rec_64 = cv2pyrUp(rec_32) + gradient_64 * (1 - gradient_mask_64[:, :, None])
+    rec_128 = cv2pyrUp(rec_64) + gradient_128 * (1 - gradient_mask_128[:, :, None])
+    rec_256 = cv2pyrUp(rec_128) + gradient_256 * (1 - gradient_mask_256[:, :, None])
+    rec_512 = cv2pyrUp(rec_256) + gradient_512 * (1 - gradient_mask_512[:, :, None])
+    rec_1024 = cv2pyrUp(rec_512) + gradient_1024 * (1 - gradient_mask_1024[:, :, None])
+
+    return rec_1024.clip(0, 255).astype(np.uint8)
+
+
+def tiny_deatlize(visulized, s2048):
+    gradient_mask_2048 = s2048.copy()
+    gradient_mask_1024 = np_min_pool(gradient_mask_2048)
+    gradient_mask_512 = np_min_pool(gradient_mask_1024)
+    gradient_mask_256 = np_min_pool(gradient_mask_512)
+
+    gradient_mask_2048 = low_down(gradient_mask_2048)
+    gradient_mask_1024 = low_down(gradient_mask_1024)
+    gradient_mask_512 = low_down(gradient_mask_512)
+    gradient_mask_256 = low_down(gradient_mask_256)
+
+    sample_2048 = visulized.astype(np.float32)
+    sample_1024 = cv2.pyrDown(sample_2048)
+    sample_512 = cv2.pyrDown(sample_1024)
+    sample_256 = cv2.pyrDown(sample_512)
+    sample_128 = cv2.pyrDown(sample_256)
+
+    gradient_2048 = sample_2048 - cv2.pyrUp(sample_1024)
+    gradient_1024 = sample_1024 - cv2.pyrUp(sample_512)
+    gradient_512 = sample_512 - cv2.pyrUp(sample_256)
+    gradient_256 = sample_256 - cv2.pyrUp(sample_128)
+
+    rec_128 = sample_128
+    rec_256 = cv2.pyrUp(rec_128) + gradient_256 * (1 - gradient_mask_256[:, :, None])
+    rec_512 = cv2.pyrUp(rec_256) + gradient_512 * (1 - gradient_mask_512[:, :, None])
+    rec_1024 = cv2.pyrUp(rec_512) + gradient_1024 * (1 - gradient_mask_1024[:, :, None])
+    rec_2048 = cv2.pyrUp(rec_1024) + gradient_2048 * (1 - gradient_mask_2048[:, :, None])
+    return rec_2048.clip(0, 255).astype(np.uint8)
+
+
+def adain(x, y):
+    x_high = cv2.GaussianBlur(x, (0, 0), 3.0)
+    y_high = cv2.GaussianBlur(y, (0, 0), 3.0)
+    return (x.astype(np.float32) - x_high.astype(np.float32) + y_high.astype(np.float32)).clip(0, 255).astype(np.uint8)
+
+
+def corrupt(x, b128):
+    float_sketch = x.astype(float)
+    float_base = cv2.resize(float_sketch, (b128.shape[1], b128.shape[0]), cv2.INTER_AREA)
+    alpha = b128[:, :, 0] / 255.0
+    float_base = alpha * float_base + (1 - alpha) * np.mean(float_base)
+    float_base = cv2.GaussianBlur(float_base, (0, 0), 8.0)
+    float_base = cv2.resize(float_base, (x.shape[1], x.shape[0]), cv2.INTER_CUBIC)
+    result = float_sketch / (float_base + 1e-10)
+    result = result.clip(0, 1)
+    result -= np.min(result)
+    result /= np.max(result)
+    return (result * 255.0).clip(0, 255).astype(np.uint8)
+
+
+def fuse_sketch(color, sketch, fills, fixer, points_arr, colors_arr):
+    sketch = cv2.resize(sketch, (color.shape[1], color.shape[0]))
+    fills = cv2.resize(fills, (color.shape[1], color.shape[0]), interpolation=cv2.INTER_NEAREST)
+    fill_id = np.unique(fills.flatten())
+    bg = np.zeros_like(color, dtype=np.uint8)
+    checking_result = np.zeros(dtype=np.int32, shape=(np.max(fills) + 1,)) - 1
+    length_points = int(len(points_arr))
+    for _ in range(length_points):
+        checking_result[fills[points_arr[_][0], points_arr[_][1]]] = _
+    for id in fill_id:
+        points = np.where(fills == id)
+        if len(points[0]) > 0:
+            color_id = checking_result[id]
+            if color_id > -1:
+                bg[points] = np.array(colors_arr[color_id])
+            else:
+                bg[points] = np.median(color[points], axis=0)
+    fixed = adain(fixer(sketch, bg), bg)
+    result = (fixed.astype(np.float32) + sketch[:, :, None].astype(np.float32) - 255.0).clip(0, 255).astype(np.uint8)
+    return result, fixed, bg
+
+
+def balance_fill(color, fills, points, sizer):
+    color = cv2.resize(color, (sizer.shape[1], sizer.shape[0]), interpolation=cv2.INTER_NEAREST)
+    points = cv2.resize(points, (sizer.shape[1], sizer.shape[0]), interpolation=cv2.INTER_NEAREST)
+    bg = np.zeros_like(color, dtype=np.uint8)
+    for region in fills:
+        if len(region[0]) > 0:
+            region_points = points[region]
+            region_points = region_points[region_points[:, 3] > 0]
+            if region_points.shape[0] > 0:
+                points_color, points_color_count = np.unique(region_points, return_counts=True, axis=0)
+                bg[region] = points_color[np.argmax(points_color_count)][0:3]
+            else:
+                bg[region] = np.median(color[region], axis=0)
+    return bg
+
+
+def shade_fill(color, fills, points, sizer):
+    color = cv2.resize(color, (sizer.shape[1], sizer.shape[0]), interpolation=cv2.INTER_NEAREST)
+    points = cv2.resize(points, (sizer.shape[1], sizer.shape[0]), interpolation=cv2.INTER_NEAREST)
+    bg = np.zeros_like(color, dtype=np.uint8)
+    for region in fills:
+        if len(region[0]) > 0:
+            region_points = points[region]
+            region_points = region_points[region_points[:, 3] > 0]
+            if region_points.shape[0] > 0:
+                points_color, points_color_count = np.unique(region_points, return_counts=True, axis=0)
+                c = points_color[np.argmax(points_color_count)][0:3]
+                r = c[0]
+                g = c[1]
+                b = c[2]
+                if r == 1 and g == 233 and b == 0:
+                    bg[region] = 255
+                elif r == 0 and g == 233 and b == 1:
+                    bg[region] = 0
+                else:
+                    bg[region] = np.median(color[region], axis=0)
+            else:
+                bg[region] = np.median(color[region], axis=0)
+    return bg
+
+
+def get_alpha_piece(points):
+    padded_points = np.pad(points, [[1, 1], [1, 1], [0, 0]], 'constant', constant_values=127)
+    lines = 255 - padded_points[:, :, 3]
+    lines[lines < 240] = 0
+    fills = flood_fill_multi(lines, merge=True)
+    result = np.zeros_like(padded_points)
+    for item in fills:
+        points0 = padded_points[(item[0], item[1] + 1)]
+        points1 = padded_points[(item[0], item[1] - 1)]
+        points2 = padded_points[(item[0] + 1, item[1])]
+        points3 = padded_points[(item[0] - 1, item[1])]
+        all_points = np.concatenate([points0, points1, points2, points3], axis=0)
+        all_points = all_points[all_points[:, 3] > 0]
+        all_points = np.unique(all_points, axis=0)
+        if all_points.shape[0] == 1:
+            result[item] = all_points[0]
+    piece = result[1:-1, 1:-1, :]
+    piece = np.maximum(piece, points)
+    return piece, points
+
+
+def fin_deatlize(color, sketch):
+
+    cf = color.astype(np.float32)
+    alpha = sketch.astype(np.float32)[:, :, None] / 255.0
+
+    plain = cf * alpha
+    lines = cf * (1 - alpha)
+
+    plain = cv2.medianBlur(plain, 5)
+    plain = cv2.medianBlur(plain, 3)
+
+    fin = plain + lines
+
+    return fin.clip(0, 255).astype(np.uint8)
+

linefiller/trappedball_fill.py

@@ -0,0 +1,436 @@
+import cv2
+import numpy as np
+from scipy.ndimage import label
+from numba import njit
+
+
+def get_ball_structuring_element(radius):
+    """Get a ball shape structuring element with specific radius for morphology operation.
+    The radius of ball usually equals to (leaking_gap_size / 2).
+    
+    # Arguments
+        radius: radius of ball shape.
+             
+    # Returns
+        an array of ball structuring element.
+    """
+    return cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2 * radius + 1, 2 * radius + 1))
+
+
+def get_unfilled_point(image):
+    """Get points belong to unfilled(value==255) area.
+
+    # Arguments
+        image: an image.
+
+    # Returns
+        an array of points.
+    """
+    y, x = np.where(image == 255)
+
+    return np.stack((x.astype(int), y.astype(int)), axis=-1)
+
+
+def exclude_area(image, radius):
+    """Perform erosion on image to exclude points near the boundary.
+    We want to pick part using floodfill from the seed point after dilation. 
+    When the seed point is near boundary, it might not stay in the fill, and would
+    not be a valid point for next floodfill operation. So we ignore these points with erosion.
+
+    # Arguments
+        image: an image.
+        radius: radius of ball shape.
+
+    # Returns
+        an image after dilation.
+    """
+    return cv2.morphologyEx(image, cv2.MORPH_ERODE, get_ball_structuring_element(radius), anchor=(-1, -1), iterations=1)
+
+
+def trapped_ball_fill_single(image, seed_point, radius):
+    """Perform a single trapped ball fill operation.
+
+    # Arguments
+        image: an image. the image should consist of white background, black lines and black fills.
+               the white area is unfilled area, and the black area is filled area.
+        seed_point: seed point for trapped-ball fill, a tuple (integer, integer).
+        radius: radius of ball shape.
+    # Returns
+        an image after filling.
+    """
+    ball = get_ball_structuring_element(radius)
+
+    pass1 = np.full(image.shape, 255, np.uint8)
+    pass2 = np.full(image.shape, 255, np.uint8)
+
+    im_inv = cv2.bitwise_not(image)
+
+    # Floodfill the image
+    mask1 = cv2.copyMakeBorder(im_inv, 1, 1, 1, 1, cv2.BORDER_CONSTANT, 0)
+    _, pass1, _, _ = cv2.floodFill(pass1, mask1, seed_point, 0, 0, 0, 4)
+
+    # Perform dilation on image. The fill areas between gaps became disconnected.
+    pass1 = cv2.morphologyEx(pass1, cv2.MORPH_DILATE, ball, anchor=(-1, -1), iterations=1)
+    mask2 = cv2.copyMakeBorder(pass1, 1, 1, 1, 1, cv2.BORDER_CONSTANT, 0)
+
+    # Floodfill with seed point again to select one fill area.
+    _, pass2, _, rect = cv2.floodFill(pass2, mask2, seed_point, 0, 0, 0, 4)
+    # Perform erosion on the fill result leaking-proof fill.
+    pass2 = cv2.morphologyEx(pass2, cv2.MORPH_ERODE, ball, anchor=(-1, -1), iterations=1)
+
+    return pass2
+
+
+def trapped_ball_fill_multi(image, radius, method='mean', max_iter=1000):
+    """Perform multi trapped ball fill operations until all valid areas are filled.
+
+    # Arguments
+        image: an image. The image should consist of white background, black lines and black fills.
+               the white area is unfilled area, and the black area is filled area.
+        radius: radius of ball shape.
+        method: method for filtering the fills. 
+               'max' is usually with large radius for select large area such as background.
+        max_iter: max iteration number.
+    # Returns
+        an array of fills' points.
+    """
+    print('trapped-ball ' + str(radius))
+
+    unfill_area = image
+    filled_area, filled_area_size, result = [], [], []
+
+    for _ in range(max_iter):
+        points = get_unfilled_point(exclude_area(unfill_area, radius))
+
+        if not len(points) > 0:
+            break
+
+        fill = trapped_ball_fill_single(unfill_area, (points[0][0], points[0][1]), radius)
+        unfill_area = cv2.bitwise_and(unfill_area, fill)
+
+        filled_area.append(np.where(fill == 0))
+        filled_area_size.append(len(np.where(fill == 0)[0]))
+
+    filled_area_size = np.asarray(filled_area_size)
+
+    if method == 'max':
+        area_size_filter = np.max(filled_area_size)
+    elif method == 'median':
+        area_size_filter = np.median(filled_area_size)
+    elif method == 'mean':
+        area_size_filter = np.mean(filled_area_size)
+    else:
+        area_size_filter = 0
+
+    result_idx = np.where(filled_area_size >= area_size_filter)[0]
+
+    for i in result_idx:
+        result.append(filled_area[i])
+
+    return result
+
+
+def flood_fill_single(im, seed_point):
+    """Perform a single flood fill operation.
+
+    # Arguments
+        image: an image. the image should consist of white background, black lines and black fills.
+               the white area is unfilled area, and the black area is filled area.
+        seed_point: seed point for trapped-ball fill, a tuple (integer, integer).
+    # Returns
+        an image after filling.
+    """
+    pass1 = np.full(im.shape, 255, np.uint8)
+
+    im_inv = cv2.bitwise_not(im)
+
+    mask1 = cv2.copyMakeBorder(im_inv, 1, 1, 1, 1, cv2.BORDER_CONSTANT, 0)
+    _, pass1, _, _ = cv2.floodFill(pass1, mask1, seed_point, 0, 0, 0, 4)
+
+    return pass1
+
+
+@njit
+def count_all(labeled_array, all_counts):
+    M = labeled_array.shape[0]
+    N = labeled_array.shape[1]
+    for x in range(M):
+        for y in range(N):
+            i = labeled_array[x, y] - 1
+            if i > -1:
+                all_counts[i] = all_counts[i] + 1
+    return
+
+
+@njit
+def trace_all(labeled_array, xs, ys, cs):
+    M = labeled_array.shape[0]
+    N = labeled_array.shape[1]
+    for x in range(M):
+        for y in range(N):
+            current_label = labeled_array[x, y] - 1
+            if current_label > -1:
+                current_label_count = cs[current_label]
+                xs[current_label][current_label_count] = x
+                ys[current_label][current_label_count] = y
+                cs[current_label] = current_label_count + 1
+    return
+
+
+def find_all(labeled_array):
+    hist_size = int(np.max(labeled_array))
+    if hist_size == 0:
+        return []
+    all_counts = [0 for _ in range(hist_size)]
+    count_all(labeled_array, all_counts)
+    xs = [np.zeros(shape=(item, ), dtype=np.uint32) for item in all_counts]
+    ys = [np.zeros(shape=(item, ), dtype=np.uint32) for item in all_counts]
+    cs = [0 for item in all_counts]
+    trace_all(labeled_array, xs, ys, cs)
+    filled_area = []
+    for _ in range(hist_size):
+        filled_area.append((xs[_], ys[_]))
+    return filled_area
+
+
+def flood_fill_multi(image, merge=False):
+    print('floodfill')
+
+    labeled_array, num_features = label(image / 255)
+    print('floodfill_ok1')
+
+    filled_area = find_all(labeled_array)
+
+    print('floodfill_ok2')
+
+    if merge:
+        new_fill = []
+        for item in filled_area:
+            if len(item[0]) > 8:
+                new_fill.append(item)
+        return new_fill
+
+    print('floodfill_ok3')
+
+    return filled_area
+
+
+def old_flood_fill_multi(image, max_iter=20000):
+    """Perform multi flood fill operations until all valid areas are filled.
+    This operation will fill all rest areas, which may result large amount of fills.
+
+    # Arguments
+        image: an image. the image should contain white background, black lines and black fills.
+               the white area is unfilled area, and the black area is filled area.
+        max_iter: max iteration number.
+    # Returns
+        an array of fills' points.
+    """
+    print('floodfill')
+
+    unfill_area = image
+    filled_area = []
+
+    for _ in range(max_iter):
+        points = get_unfilled_point(unfill_area)
+
+        if not len(points) > 0:
+            break
+
+        fill = flood_fill_single(unfill_area, (points[0][0], points[0][1]))
+        unfill_area = cv2.bitwise_and(unfill_area, fill)
+
+        filled_area.append(np.where(fill == 0))
+
+    return filled_area
+
+
+def mark_fill(image, fills):
+    """Mark filled areas with 0.
+
+    # Arguments
+        image: an image.
+        fills: an array of fills' points.
+    # Returns
+        an image.
+    """
+    result = image.copy()
+
+    for fill in fills:
+        result[fill] = 0
+
+    return result
+
+
+def build_fill_map(image, fills):
+    """Make an image(array) with each pixel(element) marked with fills' id. id of line is 0.
+
+    # Arguments
+        image: an image.
+        fills: an array of fills' points.
+    # Returns
+        an array.
+    """
+    result = np.zeros(image.shape[:2], np.int)
+
+    for index, fill in enumerate(fills):
+
+        if(len(fill[0]) == 0):
+            continue
+
+        result[fill] = index + 1
+
+    return result
+
+
+def show_fill_map(fillmap):
+    """Mark filled areas with colors. It is useful for visualization.
+
+    # Arguments
+        image: an image.
+        fills: an array of fills' points.
+    # Returns
+        an image.
+    """
+    # Generate color for each fill randomly.
+    colors = np.random.randint(0, 255, (np.max(fillmap) + 1, 3))
+    # Id of line is 0, and its color is black.
+    colors[0] = [0, 0, 0]
+
+    return colors[fillmap]
+
+
+def get_bounding_rect(points):
+    """Get a bounding rect of points.
+
+    # Arguments
+        points: array of points.
+    # Returns
+        rect coord
+    """
+    x1, y1, x2, y2 = np.min(points[1]), np.min(points[0]), np.max(points[1]), np.max(points[0])
+    return x1, y1, x2, y2
+
+
+def get_border_bounding_rect(h, w, p1, p2, r):
+    """Get a valid bounding rect in the image with border of specific size.
+
+    # Arguments
+        h: image max height.
+        w: image max width.
+        p1: start point of rect.
+        p2: end point of rect.
+        r: border radius.
+    # Returns
+        rect coord
+    """
+    x1, y1, x2, y2 = p1[0], p1[1], p2[0], p2[1]
+
+    x1 = x1 - r if 0 < x1 - r else 0
+    y1 = y1 - r if 0 < y1 - r else 0
+    x2 = x2 + r + 1 if x2 + r + 1 < w else w
+    y2 = y2 + r + 1 if y2 + r + 1 < h else h
+
+    return x1, y1, x2, y2
+
+
+def get_border_point(points, rect, max_height, max_width):
+    """Get border points of a fill area
+
+    # Arguments
+        points: points of fill .
+        rect: bounding rect of fill.
+        max_height: image max height.
+        max_width: image max width.
+    # Returns
+        points , convex shape of points
+    """
+    # Get a local bounding rect.
+    border_rect = get_border_bounding_rect(max_height, max_width, rect[:2], rect[2:], 2)
+
+    # Get fill in rect.
+    fill = np.zeros((border_rect[3] - border_rect[1], border_rect[2] - border_rect[0]), np.uint8)
+    # Move points to the rect.
+    fill[(points[0] - border_rect[1], points[1] - border_rect[0])] = 255
+
+    # Get shape.
+    _, contours, _ = cv2.findContours(fill, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    # approx_shape = cv2.approxPolyDP(contours[0], 0.02 * cv2.arcLength(contours[0], True), True)
+
+    # Get border pixel.
+    # Structuring element in cross shape is used instead of box to get 4-connected border.
+    cross = cv2.getStructuringElement(cv2.MORPH_CROSS, (3, 3))
+    border_pixel_mask = cv2.morphologyEx(fill, cv2.MORPH_DILATE, cross, anchor=(-1, -1), iterations=1) - fill
+    border_pixel_points = np.where(border_pixel_mask == 255)
+
+    # Transform points back to fillmap.
+    border_pixel_points = (border_pixel_points[0] + border_rect[1], border_pixel_points[1] + border_rect[0])
+
+    return border_pixel_points
+
+
+def merge_fill(fillmap, max_iter=20):
+    """Merge fill areas.
+
+    # Arguments
+        fillmap: an image.
+        max_iter: max iteration number.
+    # Returns
+        an image.
+    """
+    max_height, max_width = fillmap.shape[:2]
+    result = fillmap.copy()
+
+    for i in range(max_iter):
+        print('merge ' + str(i + 1))
+
+        result[np.where(fillmap == 0)] = 0
+
+        fill_id = np.unique(result.flatten())
+        fills = []
+
+        for j in fill_id:
+            point = np.where(result == j)
+
+            fills.append({
+                'id': j,
+                'point': point,
+                'area': len(point[0]),
+            })
+
+        for j, f in enumerate(fills):
+            # ignore lines
+            if f['id'] == 0:
+                continue
+
+            if f['area'] < 5:
+                result[f['point']] = 0
+
+        if len(fill_id) == len(np.unique(result.flatten())):
+            break
+
+    return result
+
+
+def merge_one(fillmap):
+    result = fillmap.copy()
+    print('merge')
+    result[np.where(fillmap == 0)] = 0
+    fill_id = np.unique(result.flatten())
+    fills = []
+    for j in fill_id:
+        point = np.where(result == j)
+        fills.append({
+            'id': j,
+            'point': point,
+            'area': len(point[0]),
+        })
+    for j, f in enumerate(fills):
+        # ignore lines
+        if f['id'] == 0:
+            continue
+
+        if f['area'] < 5:
+            result[f['point']] = 0
+    return result
+

models.py

@@ -0,0 +1,299 @@
+from keras.layers import Conv2D, Activation, Input, Concatenate, LeakyReLU, Lambda, AveragePooling2D, UpSampling2D, Convolution2D, BatchNormalization, Deconvolution2D, Add
+from keras.models import Model
+from InstanceNorm import InstanceNormalization
+
+
+def make_standard_UNET(channels,outs):
+
+    def relu(x):
+        return Activation('relu')(x)
+
+    def concat(x):
+        return Concatenate()(x)
+
+    c0 = Convolution2D(filters=32, kernel_size=3, strides=1, padding='same', name='c0')
+    c1 = Convolution2D(filters=64, kernel_size=4, strides=2, padding='same', name='c1')
+    c2 = Convolution2D(filters=64, kernel_size=3, strides=1, padding='same', name='c2')
+    c3 = Convolution2D(filters=128, kernel_size=4, strides=2, padding='same', name='c3')
+    c4 = Convolution2D(filters=128, kernel_size=3, strides=1, padding='same', name='c4')
+    c5 = Convolution2D(filters=256, kernel_size=4, strides=2, padding='same', name='c5')
+    c6 = Convolution2D(filters=256, kernel_size=3, strides=1, padding='same', name='c6')
+    c7 = Convolution2D(filters=512, kernel_size=4, strides=2, padding='same', name='c7')
+    c8 = Convolution2D(filters=512, kernel_size=3, strides=1, padding='same', name='c8')
+
+    bnc0 = BatchNormalization(axis=3, name='bnc0')
+    bnc1 = BatchNormalization(axis=3, name='bnc1')
+    bnc2 = BatchNormalization(axis=3, name='bnc2')
+    bnc3 = BatchNormalization(axis=3, name='bnc3')
+    bnc4 = BatchNormalization(axis=3, name='bnc4')
+    bnc5 = BatchNormalization(axis=3, name='bnc5')
+    bnc6 = BatchNormalization(axis=3, name='bnc6')
+    bnc7 = BatchNormalization(axis=3, name='bnc7')
+    bnc8 = BatchNormalization(axis=3, name='bnc8')
+
+    dc8 = Deconvolution2D(filters=512, kernel_size=4, strides=2, padding='same', name='dc8_')
+    dc7 = Convolution2D(filters=256, kernel_size=3, strides=1, padding='same', name='dc7')
+    dc6 = Deconvolution2D(filters=256, kernel_size=4, strides=2, padding='same', name='dc6_')
+    dc5 = Convolution2D(filters=128, kernel_size=3, strides=1, padding='same', name='dc5')
+    dc4 = Deconvolution2D(filters=128, kernel_size=4, strides=2, padding='same', name='dc4_')
+    dc3 = Convolution2D(filters=64, kernel_size=3, strides=1, padding='same', name='dc3')
+    dc2 = Deconvolution2D(filters=64, kernel_size=4, strides=2, padding='same', name='dc2_')
+    dc1 = Convolution2D(filters=32, kernel_size=3, strides=1, padding='same', name='dc1')
+    dc0 = Convolution2D(filters=outs, kernel_size=3, strides=1, padding='same', name='dc0')
+
+    bnd1 = BatchNormalization(axis=3, name='bnd1')
+    bnd2 = BatchNormalization(axis=3, name='bnd2')
+    bnd3 = BatchNormalization(axis=3, name='bnd3')
+    bnd4 = BatchNormalization(axis=3, name='bnd4')
+    bnd5 = BatchNormalization(axis=3, name='bnd5')
+    bnd6 = BatchNormalization(axis=3, name='bnd6')
+    bnd7 = BatchNormalization(axis=3, name='bnd7')
+    bnd8 = BatchNormalization(axis=3, name='bnd8')
+
+    x = Input(shape=(128, 128, channels))
+
+    e0 = relu(bnc0(c0(x), training = False))
+    e1 = relu(bnc1(c1(e0), training = False))
+    e2 = relu(bnc2(c2(e1), training = False))
+    e3 = relu(bnc3(c3(e2), training = False))
+    e4 = relu(bnc4(c4(e3), training = False))
+    e5 = relu(bnc5(c5(e4), training = False))
+    e6 = relu(bnc6(c6(e5), training = False))
+    e7 = relu(bnc7(c7(e6), training = False))
+    e8 = relu(bnc8(c8(e7), training = False))
+
+    d8 = relu(bnd8(dc8(concat([e7, e8])), training = False))
+    d7 = relu(bnd7(dc7(d8), training = False))
+    d6 = relu(bnd6(dc6(concat([e6, d7])), training = False))
+    d5 = relu(bnd5(dc5(d6), training = False))
+    d4 = relu(bnd4(dc4(concat([e4, d5])), training = False))
+    d3 = relu(bnd3(dc3(d4), training = False))
+    d2 = relu(bnd2(dc2(concat([e2, d3])), training = False))
+    d1 = relu(bnd1(dc1(d2), training = False))
+    d0 = dc0(concat([e0, d1]))
+
+    model = Model(inputs=x,outputs=d0)
+
+    return model
+
+
+def make_diff_net():
+
+    def conv(x, filters, name):
+        return Conv2D(filters=filters, strides=(1, 1), kernel_size=(3, 3), padding='same', name=name)(x)
+
+    def relu(x):
+        return Activation('relu')(x)
+
+    def lrelu(x):
+        return LeakyReLU(alpha=0.1)(x)
+
+    def r_block(x, filters, name=None):
+        return relu(conv(relu(conv(x, filters, None if name is None else name + '_c1')), filters,
+                         None if name is None else name + '_c2'))
+
+    def cat(a, b):
+        return Concatenate()([UpSampling2D((2, 2))(a), b])
+
+    def dog(x):
+        down = AveragePooling2D((2, 2))(x)
+        up = UpSampling2D((2, 2))(down)
+        diff = Lambda(lambda p: p[0] - p[1])([x, up])
+        return down, diff
+
+    ip = Input(shape=(512, 512, 3))
+
+    c512 = r_block(ip, 16, 'c512')
+
+    c256, l512 = dog(c512)
+    c256 = r_block(c256, 32, 'c256')
+
+    c128, l256 = dog(c256)
+    c128 = r_block(c128, 64, 'c128')
+
+    c64, l128 = dog(c128)
+    c64 = r_block(c64, 128, 'c64')
+
+    c32, l64 = dog(c64)
+    c32 = r_block(c32, 256, 'c32')
+
+    c16, l32 = dog(c32)
+    c16 = r_block(c16, 512, 'c16')
+
+    d32 = cat(c16, l32)
+    d32 = r_block(d32, 256, 'd32')
+
+    d64 = cat(d32, l64)
+    d64 = r_block(d64, 128, 'd64')
+
+    d128 = cat(d64, l128)
+    d128 = r_block(d128, 64, 'd128')
+
+    d256 = cat(d128, l256)
+    d256 = r_block(d256, 32, 'd256')
+
+    d512 = cat(d256, l512)
+    d512 = r_block(d512, 16, 'd512')
+
+    op = conv(d512, 1, 'op')
+
+    return Model(inputs=ip, outputs=op)
+
+
+def make_wnet256():
+
+    def conv(x, filters):
+        return Conv2D(filters=filters, strides=(1, 1), kernel_size=(3, 3), padding='same')(x)
+
+    def relu(x):
+        return Activation('relu')(x)
+
+    def lrelu(x):
+        return LeakyReLU(alpha=0.1)(x)
+
+    def r_block(x, filters):
+        return relu(conv(relu(conv(x, filters)), filters))
+
+    def res_block(x, filters):
+        return relu(Add()([x, conv(relu(conv(x, filters)), filters)]))
+
+    def cat(a, b):
+        return Concatenate()([UpSampling2D((2, 2))(a), b])
+
+    def dog(x):
+        down = AveragePooling2D((2, 2))(x)
+        up = UpSampling2D((2, 2))(down)
+        diff = Lambda(lambda p: p[0] - p[1])([x, up])
+        return down, diff
+
+    ip_sketch = Input(shape=(256, 256, 1))
+    ip_color = Input(shape=(256, 256, 3))
+
+    c256 = r_block(ip_sketch, 32)
+
+    c128, l256 = dog(c256)
+    c128 = r_block(c128, 64)
+
+    c64, l128 = dog(c128)
+    c64 = r_block(c64, 128)
+
+    c32, l64 = dog(c64)
+    c32 = r_block(Concatenate()([c32, AveragePooling2D((8, 8))(ip_color)]), 256)
+
+    c32 = res_block(c32, 256)
+    c32 = res_block(c32, 256)
+    c32 = res_block(c32, 256)
+
+    c32 = res_block(c32, 256)
+    c32 = res_block(c32, 256)
+    c32 = res_block(c32, 256)
+
+    c32 = res_block(c32, 256)
+    c32 = res_block(c32, 256)
+    c32 = res_block(c32, 256)
+
+    d64 = cat(c32, l64)
+    d64 = r_block(d64, 128)
+
+    d128 = cat(d64, l128)
+    d128 = r_block(d128, 64)
+
+    d256 = cat(d128, l256)
+    d256 = r_block(d256, 32)
+
+    op = conv(d256, 3)
+
+    return Model(inputs=[ip_sketch, ip_color], outputs=op)
+
+
+def make_unet512():
+
+    def conv(x, filters, strides=(1, 1), kernel_size=(3, 3)):
+        return Conv2D(filters=filters, strides=strides, kernel_size=kernel_size, padding='same')(x)
+
+    def donv(x, filters, strides=(2, 2), kernel_size=(4, 4)):
+        return Deconvolution2D(filters=filters, strides=strides, kernel_size=kernel_size, padding='same')(x)
+
+    def relu(x):
+        return Activation('relu')(x)
+
+    def sigmoid(x):
+        return Activation('sigmoid')(x)
+
+    def norm(x):
+        return InstanceNormalization(axis=3)(x)
+
+    def cat(a, b):
+        return Concatenate()([a, b])
+
+    def res(x, filters):
+        c1 = relu(norm(conv(x, filters // 2)))
+        c2 = norm(conv(c1, filters))
+        ad = Add()([x, c2])
+        return relu(ad)
+
+    ip = Input(shape=(512, 512, 3))
+
+    c512 = relu(norm(conv(ip, 16, strides=(1, 1), kernel_size=(3, 3))))
+    c256 = relu(norm(conv(c512, 32, strides=(2, 2), kernel_size=(4, 4))))
+
+    c128 = relu(norm(conv(c256, 64, strides=(2, 2), kernel_size=(4, 4))))
+    c128 = res(c128, 64)
+
+    c64 = relu(norm(conv(c128, 128, strides=(2, 2), kernel_size=(4, 4))))
+    c64 = res(c64, 128)
+    c64 = res(c64, 128)
+
+    c32 = relu(norm(conv(c64, 256, strides=(2, 2), kernel_size=(4, 4))))
+    c32 = res(c32, 256)
+    c32 = res(c32, 256)
+    c32 = res(c32, 256)
+    c32 = res(c32, 256)
+    c32 = res(c32, 256)
+    c32 = res(c32, 256)
+    c32 = res(c32, 256)
+    c32 = res(c32, 256)
+
+    c16 = relu(norm(conv(c32, 512, strides=(2, 2), kernel_size=(4, 4))))
+    c16 = res(c16, 512)
+    c16 = res(c16, 512)
+    c16 = res(c16, 512)
+    c16 = res(c16, 512)
+    c16 = res(c16, 512)
+    c16 = res(c16, 512)
+    c16 = res(c16, 512)
+    c16 = res(c16, 512)
+
+    c8 = relu(norm(conv(c16, 1024, strides=(2, 2), kernel_size=(4, 4))))
+    c8 = res(c8, 1024)
+    c8 = res(c8, 1024)
+    c8 = res(c8, 1024)
+    c8 = res(c8, 1024)
+
+    e16 = relu(norm(donv(c8, 512, strides=(2, 2), kernel_size=(4, 4))))
+    e16 = cat(e16, c16)
+    e16 = relu(norm(conv(e16, 512, strides=(1, 1), kernel_size=(3, 3))))
+
+    e32 = relu(norm(donv(e16, 256, strides=(2, 2), kernel_size=(4, 4))))
+    e32 = cat(e32, c32)
+    e32 = relu(norm(conv(e32, 256, strides=(1, 1), kernel_size=(3, 3))))
+
+    e64 = relu(norm(donv(e32, 128, strides=(2, 2), kernel_size=(4, 4))))
+    e64 = cat(e64, c64)
+    e64 = relu(norm(conv(e64, 128, strides=(1, 1), kernel_size=(3, 3))))
+
+    e128 = relu(norm(donv(e64, 64, strides=(2, 2), kernel_size=(4, 4))))
+    e128 = cat(e128, c128)
+    e128 = relu(norm(conv(e128, 64, strides=(1, 1), kernel_size=(3, 3))))
+
+    e256 = relu(norm(donv(e128, 32, strides=(2, 2), kernel_size=(4, 4))))
+    e256 = cat(e256, c256)
+    e256 = relu(norm(conv(e256, 32, strides=(1, 1), kernel_size=(3, 3))))
+
+    e512 = relu(norm(donv(e256, 16, strides=(2, 2), kernel_size=(4, 4))))
+    e512 = cat(e512, c512)
+    e512 = relu(norm(conv(e512, 16, strides=(1, 1), kernel_size=(3, 3))))
+
+    ot = sigmoid(conv(e512, 1))
+
+    return Model(inputs=ip, outputs=ot)

nets/head.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd5f608bf5511293cd582e6d87e55f483259b8d606feba7a5042382a19cde630
+size 411622416

nets/inception.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a503841e955ee56f2e0b71219952f0f554c9eaff9e887f82853f600f0ddee80
+size 41501888

nets/mat.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2998a3cd6446ad62d87259dd5b279e344a9ba8daaf2c3c88f59ebb5ddfe09cd
+size 362262288

nets/neck.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e91e4e688379dc95a14ea8f33fa1c2141584b0387e16be5efcc7acb5299ad5d7
+size 411623808

nets/norm.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08d230c0a595dc21d2924911b2d2aa9f334c2180a6526d947d11a4d00e5ce4d0
+size 113030088

nets/reader.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a65a53091acb96dbdcf325afb1b233ff374e220cb822fe8b1771dcd65f999df
+size 41502832

nets/render_head.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cacac78d2e19bd38b2efbdd8eb372594e333e5284e9f2c2b605652c8dd26ffad
+size 411612752

nets/render_neck.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66050656839363d7977dd970cc2f6d7f81722a8cf5cd67a1793bae8214a07c0b
+size 411613336

nets/tail.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2255073ccdc311d8c61c074ffeb4ad7db200ca9116011e1cf213d6d4b1967e15
+size 4018208

nets/vector.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2738164810e2c998c4e5c7598c9ad991b9aad6c300d075119da9e4201212066a
+size 31618276

nets/vgg7.net

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84539cc2f46a0937a31ff770af592242e1e2a06f9f3c3fd285ab596a7ab90be6
+size 1188680