apextoaster
/
onnx-web
1
0
Fork 0
Code Actions

feat(gui): add undo button to mask canvas

This commit is contained in:
Sean Sube 2023-02-12 22:32:33 -06:00
parent 4abbb00fd0
commit bb05395563
Signed by: ssube
GPG Key ID: 3EED7B957D362AF1
5 changed files with 239 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
gui/package.json
View File

@ -15,6 +15,7 @@
"@mui/material": "^5.11.3", "@mui/material": "^5.11.3",
"@types/lodash": "^4.14.191", "@types/lodash": "^4.14.191",
"@types/node": "^18.11.18", "@types/node": "^18.11.18",
"browser-bunyan": "^1.8.0",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"noicejs": "^5.0.0-3", "noicejs": "^5.0.0-3",
"react": "^18.2.0", "react": "^18.2.0",

304
gui/src/components/input/MaskCanvas.tsx
View File

@ -1,14 +1,17 @@
import { doesExist, Maybe, mustExist } from '@apextoaster/js-utils'; import { doesExist, Maybe, mustExist } from '@apextoaster/js-utils';
import { FormatColorFill, Gradient, InvertColors } from '@mui/icons-material'; import { FormatColorFill, Gradient, InvertColors, Undo } from '@mui/icons-material';
import { Button, Stack, Typography } from '@mui/material'; import { Button, Stack, Typography } from '@mui/material';
import { throttle } from 'lodash'; import { throttle } from 'lodash';
import React, { RefObject, useContext, useEffect, useMemo, useRef, useState } from 'react'; import React, { RefObject, useContext, useEffect, useMemo, useRef, useState } from 'react';
import { useStore } from 'zustand'; import { useStore } from 'zustand';
import { createLogger } from 'browser-bunyan';
import { SAVE_TIME } from '../../config.js'; import { SAVE_TIME } from '../../config.js';
import { ConfigContext, StateContext } from '../../state.js'; import { ConfigContext, StateContext } from '../../state.js';
import { imageFromBlob } from '../../utils.js';
import { NumericField } from './NumericField'; import { NumericField } from './NumericField';
export const DRAW_TIME = 25;
export const FULL_CIRCLE = 2 * Math.PI; export const FULL_CIRCLE = 2 * Math.PI;
export const FULL_OPACITY = 1.0; export const FULL_OPACITY = 1.0;
export const MASK_OPACITY = 0.75; export const MASK_OPACITY = 0.75;
@ -45,17 +48,27 @@ export interface MaskCanvasProps {
onSave: (blob: Blob) => void; onSave: (blob: Blob) => void;
} }
const logger = createLogger({ name: 'react', level: 'debug' }); // TODO: hackeroni and cheese
export function MaskCanvas(props: MaskCanvasProps) { export function MaskCanvas(props: MaskCanvasProps) {
const { source, mask } = props; const { source, mask } = props;
const { params } = mustExist(useContext(ConfigContext)); const { params } = mustExist(useContext(ConfigContext));
function drawBuffer() { function composite() {
if (doesExist(brushRef.current) && doesExist(bufferRef.current) && doesExist(canvasRef.current)) { if (doesExist(visibleRef.current)) {
const { ctx } = getClearContext(canvasRef); const { ctx } = getClearContext(visibleRef);
ctx.globalAlpha = MASK_OPACITY;
ctx.drawImage(bufferRef.current, 0, 0);
if (maskState.current !== MASK_STATE.painting) { if (doesExist(maskRef.current)) {
ctx.globalAlpha = MASK_OPACITY;
ctx.drawImage(maskRef.current, 0, 0);
}
if (doesExist(bufferRef.current)) {
ctx.globalAlpha = MASK_OPACITY;
ctx.drawImage(bufferRef.current, 0, 0);
}
if (doesExist(brushRef.current) && maskState.current !== MASK_STATE.painting) {
ctx.drawImage(brushRef.current, 0, 0); ctx.drawImage(brushRef.current, 0, 0);
} }
} }
@ -70,73 +83,94 @@ export function MaskCanvas(props: MaskCanvasProps) {
y: point.y, y: point.y,
}, brush.size); }, brush.size);
drawBuffer(); composite();
} }
function drawClicks(): void { function drawClicks(c2: Array<Point>, set: (value: React.SetStateAction<Array<Point>>) => void): boolean {
if (clicks.length > 0) { if (c2.length > 0) {
logger.debug('drawing clicks', { count: c2.length });
const { ctx } = getContext(bufferRef); const { ctx } = getContext(bufferRef);
ctx.fillStyle = grayToRGB(brush.color, brush.strength); ctx.fillStyle = grayToRGB(brush.color, brush.strength);
for (const click of clicks) { for (const click of c2) {
drawCircle(ctx, click, brush.size); drawCircle(ctx, click, brush.size);
} }
clicks.length = 0; composite();
set([]);
drawBuffer(); return true;
} }
return false;
} }
function drawSource(file: Blob): void { async function drawMask(file: Blob): Promise<void> {
const image = new Image(); const image = await imageFromBlob(file);
image.onload = () => { logger.debug('draw mask');
const { canvas, ctx } = getClearContext(bufferRef);
ctx.globalAlpha = FULL_OPACITY;
ctx.drawImage(image, 0, 0, canvas.width, canvas.height);
URL.revokeObjectURL(src); const { canvas, ctx } = getClearContext(maskRef);
ctx.globalAlpha = FULL_OPACITY;
ctx.drawImage(image, 0, 0, canvas.width, canvas.height);
drawBuffer(); // getClearContext(bufferRef);
}; composite();
const src = URL.createObjectURL(file);
image.src = src;
} }
function finishPainting() { function finishPainting() {
logger.debug('finish painting');
if (doesExist(brushRef.current)) { if (doesExist(brushRef.current)) {
getClearContext(brushRef); getClearContext(brushRef);
} }
drawClicks(); if (drawClicks(clicks, setClicks) === false) {
logger.debug('force compositing');
composite();
}
if (maskState.current === MASK_STATE.painting) { if (maskState.current === MASK_STATE.painting) {
maskState.current = MASK_STATE.dirty; maskState.current = MASK_STATE.dirty;
} }
} }
function flushBuffer(): void {
if (doesExist(maskRef.current) && doesExist(bufferRef.current)) {
logger.debug('flush buffer');
const { ctx } = getContext(maskRef);
ctx.drawImage(bufferRef.current, 0, 0);
getClearContext(bufferRef);
composite();
}
}
function saveMask(): void { function saveMask(): void {
if (doesExist(bufferRef.current)) { if (doesExist(maskRef.current)) {
logger.debug('save mask');
if (maskState.current === MASK_STATE.clean) { if (maskState.current === MASK_STATE.clean) {
return; return;
} }
bufferRef.current.toBlob((blob) => { maskRef.current.toBlob((blob) => {
maskState.current = MASK_STATE.clean; maskState.current = MASK_STATE.clean;
props.onSave(mustExist(blob)); props.onSave(mustExist(blob));
}); });
} }
} }
const save = useMemo(() => throttle(saveMask, SAVE_TIME), []); const draw = useMemo(() => throttle(drawClicks, DRAW_TIME), []);
const save = useMemo(() => throttle(saveMask, SAVE_TIME, {
trailing: true,
}), []);
// eslint-disable-next-line no-null/no-null // eslint-disable-next-line no-null/no-null
const brushRef = useRef<HTMLCanvasElement>(null); const brushRef = useRef<HTMLCanvasElement>(null);
// eslint-disable-next-line no-null/no-null // eslint-disable-next-line no-null/no-null
const bufferRef = useRef<HTMLCanvasElement>(null); const bufferRef = useRef<HTMLCanvasElement>(null);
// eslint-disable-next-line no-null/no-null // eslint-disable-next-line no-null/no-null
const canvasRef = useRef<HTMLCanvasElement>(null); const maskRef = useRef<HTMLCanvasElement>(null);
// eslint-disable-next-line no-null/no-null
const visibleRef = useRef<HTMLCanvasElement>(null);
// painting state // painting state
const maskState = useRef(MASK_STATE.clean); const maskState = useRef(MASK_STATE.clean);
@ -152,11 +186,17 @@ export function MaskCanvas(props: MaskCanvasProps) {
if (maskState.current === MASK_STATE.dirty) { if (maskState.current === MASK_STATE.dirty) {
save(); save();
} }
return () => {
logger.debug('save cleanup');
};
}, [maskState.current]); }, [maskState.current]);
useEffect(() => { useEffect(() => {
if (doesExist(bufferRef.current) && doesExist(mask)) { if (doesExist(bufferRef.current) && doesExist(mask)) {
drawSource(mask); drawMask(mask).catch((err) => {
// TODO: handle
});
} }
}, [mask]); }, [mask]);
@ -177,7 +217,9 @@ export function MaskCanvas(props: MaskCanvasProps) {
}, [source]); }, [source]);
// last resort to draw lost clicks // last resort to draw lost clicks
drawClicks(); // const lostClicks = drawClicks();
logger.debug('rendered', { clicks: clicks.length });
draw(clicks, setClicks);
const styles: React.CSSProperties = { const styles: React.CSSProperties = {
backgroundPosition: 'top left', backgroundPosition: 'top left',
@ -210,12 +252,21 @@ export function MaskCanvas(props: MaskCanvasProps) {
}} }}
/> />
<canvas <canvas
ref={canvasRef} ref={maskRef}
height={params.height.default}
width={params.width.default}
style={{
display: 'none',
}}
/>
<canvas
ref={visibleRef}
height={params.height.default} height={params.height.default}
width={params.width.default} width={params.width.default}
style={styles} style={styles}
onClick={(event) => { onClick={(event) => {
const canvas = mustExist(canvasRef.current); logger.debug('mouse click', { state: maskState.current, clicks: clicks.length });
const canvas = mustExist(visibleRef.current);
const bounds = canvas.getBoundingClientRect(); const bounds = canvas.getBoundingClientRect();
setClicks([...clicks, { setClicks([...clicks, {
@ -223,17 +274,20 @@ export function MaskCanvas(props: MaskCanvasProps) {
y: event.clientY - bounds.top, y: event.clientY - bounds.top,
}]); }]);
drawClicks(); drawClicks(clicks, setClicks);
maskState.current = MASK_STATE.dirty; maskState.current = MASK_STATE.dirty;
}} }}
onMouseDown={() => { onMouseDown={() => {
logger.debug('mouse down', { state: maskState.current, clicks: clicks.length });
maskState.current = MASK_STATE.painting; maskState.current = MASK_STATE.painting;
flushBuffer();
}} }}
onMouseLeave={finishPainting} onMouseLeave={finishPainting}
onMouseOut={finishPainting} onMouseOut={finishPainting}
onMouseUp={finishPainting} onMouseUp={finishPainting}
onMouseMove={(event) => { onMouseMove={(event) => {
const canvas = mustExist(canvasRef.current); const canvas = mustExist(visibleRef.current);
const bounds = canvas.getBoundingClientRect(); const bounds = canvas.getBoundingClientRect();
if (maskState.current === MASK_STATE.painting) { if (maskState.current === MASK_STATE.painting) {
@ -253,88 +307,100 @@ export function MaskCanvas(props: MaskCanvasProps) {
Black pixels in the mask will stay the same, white pixels will be replaced. The masked pixels will be blended Black pixels in the mask will stay the same, white pixels will be replaced. The masked pixels will be blended
with the noise source before the diffusion model runs, giving it more variety to use. with the noise source before the diffusion model runs, giving it more variety to use.
</Typography> </Typography>
<Stack direction='row' spacing={4}> <Stack>
<NumericField <Stack direction='row' spacing={4}>
label='Brush Color' <NumericField
min={COLORS.black} label='Brush Color'
max={COLORS.white} min={COLORS.black}
step={1} max={COLORS.white}
value={brush.color} step={1}
onChange={(color) => { value={brush.color}
setBrush({ color }); onChange={(color) => {
}} setBrush({ color });
/> }}
<NumericField />
label='Brush Size' <NumericField
min={1} label='Brush Size'
max={64} min={1}
step={1} max={64}
value={brush.size} step={1}
onChange={(size) => { value={brush.size}
setBrush({ size }); onChange={(size) => {
}} setBrush({ size });
/> }}
<NumericField />
decimal <NumericField
label='Brush Strength' decimal
min={0} label='Brush Strength'
max={1} min={0}
step={0.01} max={1}
value={brush.strength} step={0.01}
onChange={(strength) => { value={brush.strength}
setBrush({ strength }); onChange={(strength) => {
}} setBrush({ strength });
/> }}
<Button />
variant='outlined' </Stack>
startIcon={<FormatColorFill />} <Stack direction='row' spacing={2}>
onClick={() => { <Button
floodCanvas(bufferRef, floodBlack); variant='outlined'
drawBuffer(); startIcon={<Undo />}
maskState.current = MASK_STATE.dirty; onClick={() => {
}}> getClearContext(bufferRef);
Fill with black composite();
</Button> }}
<Button />
variant='outlined' <Button
startIcon={<FormatColorFill />} variant='outlined'
onClick={() => { startIcon={<FormatColorFill />}
floodCanvas(bufferRef, floodWhite); onClick={() => {
drawBuffer(); floodCanvas(maskRef, floodBlack);
maskState.current = MASK_STATE.dirty; composite();
}}> maskState.current = MASK_STATE.dirty;
Fill with white }}>
</Button> Fill with black
<Button </Button>
variant='outlined' <Button
startIcon={<InvertColors />} variant='outlined'
onClick={() => { startIcon={<FormatColorFill />}
floodCanvas(bufferRef, floodInvert); onClick={() => {
drawBuffer(); floodCanvas(maskRef, floodWhite);
maskState.current = MASK_STATE.dirty; composite();
}}> maskState.current = MASK_STATE.dirty;
Invert }}>
</Button> Fill with white
<Button </Button>
variant='outlined' <Button
startIcon={<Gradient />} variant='outlined'
onClick={() => { startIcon={<InvertColors />}
floodCanvas(bufferRef, floodBelow); onClick={() => {
drawBuffer(); floodCanvas(maskRef, floodInvert);
maskState.current = MASK_STATE.dirty; composite();
}}> maskState.current = MASK_STATE.dirty;
Gray to black }}>
</Button> Invert
<Button </Button>
variant='outlined' <Button
startIcon={<Gradient />} variant='outlined'
onClick={() => { startIcon={<Gradient />}
floodCanvas(bufferRef, floodAbove); onClick={() => {
drawBuffer(); floodCanvas(maskRef, floodBelow);
maskState.current = MASK_STATE.dirty; composite();
}}> maskState.current = MASK_STATE.dirty;
Gray to white }}>
</Button> Gray to black
</Button>
<Button
variant='outlined'
startIcon={<Gradient />}
onClick={() => {
floodCanvas(maskRef, floodAbove);
composite();
maskState.current = MASK_STATE.dirty;
}}>
Gray to white
</Button>
</Stack>
</Stack> </Stack>
</Stack>; </Stack>;
} }

5
gui/src/main.tsx
View File

@ -80,6 +80,11 @@ export async function main() {
...s.upscaleTab, ...s.upscaleTab,
source: undefined, source: undefined,
}, },
blend: {
...s.blend,
mask: undefined,
sources: [],
}
}; };
}, },
storage: createJSONStorage(() => localStorage), storage: createJSONStorage(() => localStorage),

12
gui/src/utils.ts Normal file
View File

@ -0,0 +1,12 @@
export function imageFromBlob(blob: Blob): Promise<HTMLImageElement> {
return new Promise((res, rej) => {
const image = new Image();
image.onload = () => {
URL.revokeObjectURL(src);
res(image);
};
const src = URL.createObjectURL(blob);
image.src = src;
});
}

36
gui/yarn.lock
View File

@ -80,6 +80,32 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@browser-bunyan/console-formatted-stream@^1.8.0":
version "1.8.0"
resolved "https://registry.yarnpkg.com/@browser-bunyan/console-formatted-stream/-/console-formatted-stream-1.8.0.tgz#dda9dcab6ce445cbf2911045709930757e5d48c1"
integrity sha512-Lg5SC2uXrvZ6aLwLZT6SErfN1Is4NcrTOb5km4BW/BfL8Lv0CfpsYuhuD7ltdURL6awTYBUiT+BwhKw1Xd9glQ==
dependencies:
"@browser-bunyan/levels" "^1.8.0"
"@browser-bunyan/console-plain-stream@^1.8.0":
version "1.8.0"
resolved "https://registry.yarnpkg.com/@browser-bunyan/console-plain-stream/-/console-plain-stream-1.8.0.tgz#18cd8fe879a0f576cf84c4fa4647e86cd3feea3e"
integrity sha512-S0WNsH5zvMfkbayIx90wANGHQ8l3Bvd7mjgy95/bYmUzcI+Mwkv2eJcSufdTP/MbdHBhjv/lEdLDOXEPBi+w3A==
dependencies:
"@browser-bunyan/levels" "^1.8.0"
"@browser-bunyan/console-raw-stream@^1.8.0":
version "1.8.0"
resolved "https://registry.yarnpkg.com/@browser-bunyan/console-raw-stream/-/console-raw-stream-1.8.0.tgz#5d0438139bbffd9ed779241df6ae7e5f3a2a7b0c"
integrity sha512-6M/xEiNckbFslQMaS1BHAxvuvN1Wtbh/aq4UzQD3fjEPFCxtubvf4KyzwPxUXA5CXq7leVZ+cibEUCRBsm5bzg==
dependencies:
"@browser-bunyan/levels" "^1.8.0"
"@browser-bunyan/levels@^1.8.0":
version "1.8.0"
resolved "https://registry.yarnpkg.com/@browser-bunyan/levels/-/levels-1.8.0.tgz#1c0a98d04284e0620e8ee414d7ce43385080a5cf"
integrity sha512-f9oSDik8kAl+4rhVyHqIr012P1boHFUKc7D9nzA5+lDsFoP90UQnDwpseqBdF2mTaWYju10E7h+GdH8u+7MHOQ==
"@emotion/babel-plugin@^11.10.5": "@emotion/babel-plugin@^11.10.5":
version "11.10.5" version "11.10.5"
resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.10.5.tgz#65fa6e1790ddc9e23cc22658a4c5dea423c55c3c" resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.10.5.tgz#65fa6e1790ddc9e23cc22658a4c5dea423c55c3c"
@ -879,6 +905,16 @@ broadcast-channel@^3.4.1:
rimraf "3.0.2" rimraf "3.0.2"
unload "2.2.0" unload "2.2.0"
browser-bunyan@^1.8.0:
version "1.8.0"
resolved "https://registry.yarnpkg.com/browser-bunyan/-/browser-bunyan-1.8.0.tgz#6b9662fea571c642fce80ad002d62e3ea1453393"
integrity sha512-Et1TaRUm8m2oy4OTi69g0qAM8wqpofACUgkdBnj1Kq2aC8Wpl8w+lNevebPG6zKH2w0Aq+BHiAXWwjm0/QbkaQ==
dependencies:
"@browser-bunyan/console-formatted-stream" "^1.8.0"
"@browser-bunyan/console-plain-stream" "^1.8.0"
"@browser-bunyan/console-raw-stream" "^1.8.0"
"@browser-bunyan/levels" "^1.8.0"
browser-stdout@1.3.1: browser-stdout@1.3.1:
version "1.3.1" version "1.3.1"
resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60"