diff --git a/gui/package.json b/gui/package.json index ae84ce4f..92f78047 100644 --- a/gui/package.json +++ b/gui/package.json @@ -15,6 +15,7 @@ "@mui/material": "^5.11.3", "@types/lodash": "^4.14.191", "@types/node": "^18.11.18", + "browser-bunyan": "^1.8.0", "lodash": "^4.17.21", "noicejs": "^5.0.0-3", "react": "^18.2.0", diff --git a/gui/src/components/input/MaskCanvas.tsx b/gui/src/components/input/MaskCanvas.tsx index 0a3d12b7..f3e6c2a6 100644 --- a/gui/src/components/input/MaskCanvas.tsx +++ b/gui/src/components/input/MaskCanvas.tsx @@ -1,14 +1,17 @@ import { doesExist, Maybe, mustExist } from '@apextoaster/js-utils'; -import { FormatColorFill, Gradient, InvertColors } from '@mui/icons-material'; +import { FormatColorFill, Gradient, InvertColors, Undo } from '@mui/icons-material'; import { Button, Stack, Typography } from '@mui/material'; import { throttle } from 'lodash'; import React, { RefObject, useContext, useEffect, useMemo, useRef, useState } from 'react'; import { useStore } from 'zustand'; +import { createLogger } from 'browser-bunyan'; import { SAVE_TIME } from '../../config.js'; import { ConfigContext, StateContext } from '../../state.js'; +import { imageFromBlob } from '../../utils.js'; import { NumericField } from './NumericField'; +export const DRAW_TIME = 25; export const FULL_CIRCLE = 2 * Math.PI; export const FULL_OPACITY = 1.0; export const MASK_OPACITY = 0.75; @@ -45,17 +48,27 @@ export interface MaskCanvasProps { onSave: (blob: Blob) => void; } +const logger = createLogger({ name: 'react', level: 'debug' }); // TODO: hackeroni and cheese + export function MaskCanvas(props: MaskCanvasProps) { const { source, mask } = props; const { params } = mustExist(useContext(ConfigContext)); - function drawBuffer() { - if (doesExist(brushRef.current) && doesExist(bufferRef.current) && doesExist(canvasRef.current)) { - const { ctx } = getClearContext(canvasRef); - ctx.globalAlpha = MASK_OPACITY; - ctx.drawImage(bufferRef.current, 0, 0); + function composite() { + if (doesExist(visibleRef.current)) { + const { ctx } = getClearContext(visibleRef); - if (maskState.current !== MASK_STATE.painting) { + if (doesExist(maskRef.current)) { + ctx.globalAlpha = MASK_OPACITY; + ctx.drawImage(maskRef.current, 0, 0); + } + + if (doesExist(bufferRef.current)) { + ctx.globalAlpha = MASK_OPACITY; + ctx.drawImage(bufferRef.current, 0, 0); + } + + if (doesExist(brushRef.current) && maskState.current !== MASK_STATE.painting) { ctx.drawImage(brushRef.current, 0, 0); } } @@ -70,73 +83,94 @@ export function MaskCanvas(props: MaskCanvasProps) { y: point.y, }, brush.size); - drawBuffer(); + composite(); } - function drawClicks(): void { - if (clicks.length > 0) { + function drawClicks(c2: Array, set: (value: React.SetStateAction>) => void): boolean { + if (c2.length > 0) { + logger.debug('drawing clicks', { count: c2.length }); + const { ctx } = getContext(bufferRef); ctx.fillStyle = grayToRGB(brush.color, brush.strength); - for (const click of clicks) { + for (const click of c2) { drawCircle(ctx, click, brush.size); } - clicks.length = 0; - - drawBuffer(); + composite(); + set([]); + return true; } + + return false; } - function drawSource(file: Blob): void { - const image = new Image(); - image.onload = () => { - const { canvas, ctx } = getClearContext(bufferRef); - ctx.globalAlpha = FULL_OPACITY; - ctx.drawImage(image, 0, 0, canvas.width, canvas.height); + async function drawMask(file: Blob): Promise { + const image = await imageFromBlob(file); + logger.debug('draw mask'); - URL.revokeObjectURL(src); + const { canvas, ctx } = getClearContext(maskRef); + ctx.globalAlpha = FULL_OPACITY; + ctx.drawImage(image, 0, 0, canvas.width, canvas.height); - drawBuffer(); - }; - - const src = URL.createObjectURL(file); - image.src = src; + // getClearContext(bufferRef); + composite(); } function finishPainting() { + logger.debug('finish painting'); + if (doesExist(brushRef.current)) { getClearContext(brushRef); } - drawClicks(); + if (drawClicks(clicks, setClicks) === false) { + logger.debug('force compositing'); + composite(); + } if (maskState.current === MASK_STATE.painting) { maskState.current = MASK_STATE.dirty; } } + function flushBuffer(): void { + if (doesExist(maskRef.current) && doesExist(bufferRef.current)) { + logger.debug('flush buffer'); + const { ctx } = getContext(maskRef); + ctx.drawImage(bufferRef.current, 0, 0); + getClearContext(bufferRef); + composite(); + } + } + function saveMask(): void { - if (doesExist(bufferRef.current)) { + if (doesExist(maskRef.current)) { + logger.debug('save mask'); if (maskState.current === MASK_STATE.clean) { return; } - bufferRef.current.toBlob((blob) => { + maskRef.current.toBlob((blob) => { maskState.current = MASK_STATE.clean; props.onSave(mustExist(blob)); }); } } - const save = useMemo(() => throttle(saveMask, SAVE_TIME), []); + const draw = useMemo(() => throttle(drawClicks, DRAW_TIME), []); + const save = useMemo(() => throttle(saveMask, SAVE_TIME, { + trailing: true, + }), []); // eslint-disable-next-line no-null/no-null const brushRef = useRef(null); // eslint-disable-next-line no-null/no-null const bufferRef = useRef(null); // eslint-disable-next-line no-null/no-null - const canvasRef = useRef(null); + const maskRef = useRef(null); + // eslint-disable-next-line no-null/no-null + const visibleRef = useRef(null); // painting state const maskState = useRef(MASK_STATE.clean); @@ -152,11 +186,17 @@ export function MaskCanvas(props: MaskCanvasProps) { if (maskState.current === MASK_STATE.dirty) { save(); } + + return () => { + logger.debug('save cleanup'); + }; }, [maskState.current]); useEffect(() => { if (doesExist(bufferRef.current) && doesExist(mask)) { - drawSource(mask); + drawMask(mask).catch((err) => { + // TODO: handle + }); } }, [mask]); @@ -177,7 +217,9 @@ export function MaskCanvas(props: MaskCanvasProps) { }, [source]); // last resort to draw lost clicks - drawClicks(); + // const lostClicks = drawClicks(); + logger.debug('rendered', { clicks: clicks.length }); + draw(clicks, setClicks); const styles: React.CSSProperties = { backgroundPosition: 'top left', @@ -210,12 +252,21 @@ export function MaskCanvas(props: MaskCanvasProps) { }} /> + { - const canvas = mustExist(canvasRef.current); + logger.debug('mouse click', { state: maskState.current, clicks: clicks.length }); + const canvas = mustExist(visibleRef.current); const bounds = canvas.getBoundingClientRect(); setClicks([...clicks, { @@ -223,17 +274,20 @@ export function MaskCanvas(props: MaskCanvasProps) { y: event.clientY - bounds.top, }]); - drawClicks(); + drawClicks(clicks, setClicks); maskState.current = MASK_STATE.dirty; }} onMouseDown={() => { + logger.debug('mouse down', { state: maskState.current, clicks: clicks.length }); maskState.current = MASK_STATE.painting; + + flushBuffer(); }} onMouseLeave={finishPainting} onMouseOut={finishPainting} onMouseUp={finishPainting} onMouseMove={(event) => { - const canvas = mustExist(canvasRef.current); + const canvas = mustExist(visibleRef.current); const bounds = canvas.getBoundingClientRect(); if (maskState.current === MASK_STATE.painting) { @@ -253,88 +307,100 @@ export function MaskCanvas(props: MaskCanvasProps) { Black pixels in the mask will stay the same, white pixels will be replaced. The masked pixels will be blended with the noise source before the diffusion model runs, giving it more variety to use. - - { - setBrush({ color }); - }} - /> - { - setBrush({ size }); - }} - /> - { - setBrush({ strength }); - }} - /> - - - - - + + + { + setBrush({ color }); + }} + /> + { + setBrush({ size }); + }} + /> + { + setBrush({ strength }); + }} + /> + + + + + + + + ; } diff --git a/gui/src/main.tsx b/gui/src/main.tsx index 2e3ec476..f673dc60 100644 --- a/gui/src/main.tsx +++ b/gui/src/main.tsx @@ -80,6 +80,11 @@ export async function main() { ...s.upscaleTab, source: undefined, }, + blend: { + ...s.blend, + mask: undefined, + sources: [], + } }; }, storage: createJSONStorage(() => localStorage), diff --git a/gui/src/utils.ts b/gui/src/utils.ts new file mode 100644 index 00000000..6d564cc7 --- /dev/null +++ b/gui/src/utils.ts @@ -0,0 +1,12 @@ +export function imageFromBlob(blob: Blob): Promise { + return new Promise((res, rej) => { + const image = new Image(); + image.onload = () => { + URL.revokeObjectURL(src); + res(image); + }; + + const src = URL.createObjectURL(blob); + image.src = src; + }); +} diff --git a/gui/yarn.lock b/gui/yarn.lock index 05cbfdcc..547d4d24 100644 --- a/gui/yarn.lock +++ b/gui/yarn.lock @@ -80,6 +80,32 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@browser-bunyan/console-formatted-stream@^1.8.0": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@browser-bunyan/console-formatted-stream/-/console-formatted-stream-1.8.0.tgz#dda9dcab6ce445cbf2911045709930757e5d48c1" + integrity sha512-Lg5SC2uXrvZ6aLwLZT6SErfN1Is4NcrTOb5km4BW/BfL8Lv0CfpsYuhuD7ltdURL6awTYBUiT+BwhKw1Xd9glQ== + dependencies: + "@browser-bunyan/levels" "^1.8.0" + +"@browser-bunyan/console-plain-stream@^1.8.0": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@browser-bunyan/console-plain-stream/-/console-plain-stream-1.8.0.tgz#18cd8fe879a0f576cf84c4fa4647e86cd3feea3e" + integrity sha512-S0WNsH5zvMfkbayIx90wANGHQ8l3Bvd7mjgy95/bYmUzcI+Mwkv2eJcSufdTP/MbdHBhjv/lEdLDOXEPBi+w3A== + dependencies: + "@browser-bunyan/levels" "^1.8.0" + +"@browser-bunyan/console-raw-stream@^1.8.0": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@browser-bunyan/console-raw-stream/-/console-raw-stream-1.8.0.tgz#5d0438139bbffd9ed779241df6ae7e5f3a2a7b0c" + integrity sha512-6M/xEiNckbFslQMaS1BHAxvuvN1Wtbh/aq4UzQD3fjEPFCxtubvf4KyzwPxUXA5CXq7leVZ+cibEUCRBsm5bzg== + dependencies: + "@browser-bunyan/levels" "^1.8.0" + +"@browser-bunyan/levels@^1.8.0": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@browser-bunyan/levels/-/levels-1.8.0.tgz#1c0a98d04284e0620e8ee414d7ce43385080a5cf" + integrity sha512-f9oSDik8kAl+4rhVyHqIr012P1boHFUKc7D9nzA5+lDsFoP90UQnDwpseqBdF2mTaWYju10E7h+GdH8u+7MHOQ== + "@emotion/babel-plugin@^11.10.5": version "11.10.5" resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.10.5.tgz#65fa6e1790ddc9e23cc22658a4c5dea423c55c3c" @@ -879,6 +905,16 @@ broadcast-channel@^3.4.1: rimraf "3.0.2" unload "2.2.0" +browser-bunyan@^1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/browser-bunyan/-/browser-bunyan-1.8.0.tgz#6b9662fea571c642fce80ad002d62e3ea1453393" + integrity sha512-Et1TaRUm8m2oy4OTi69g0qAM8wqpofACUgkdBnj1Kq2aC8Wpl8w+lNevebPG6zKH2w0Aq+BHiAXWwjm0/QbkaQ== + dependencies: + "@browser-bunyan/console-formatted-stream" "^1.8.0" + "@browser-bunyan/console-plain-stream" "^1.8.0" + "@browser-bunyan/console-raw-stream" "^1.8.0" + "@browser-bunyan/levels" "^1.8.0" + browser-stdout@1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60"