apextoaster
/
onnx-web
1
0
Fork 0
Code Actions

feat(gui): add undo button to mask canvas

This commit is contained in:
Sean Sube 2023-02-12 22:32:33 -06:00
parent 4abbb00fd0
commit bb05395563
Signed by: ssube
GPG Key ID: 3EED7B957D362AF1
5 changed files with 239 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
gui/package.json
View File

@ -15,6 +15,7 @@
"@mui/material": "^5.11.3",
"@types/lodash": "^4.14.191",
"@types/node": "^18.11.18",
"browser-bunyan": "^1.8.0",
"lodash": "^4.17.21",
"noicejs": "^5.0.0-3",
"react": "^18.2.0",

304
gui/src/components/input/MaskCanvas.tsx
View File

@ -1,14 +1,17 @@
import { doesExist, Maybe, mustExist } from '@apextoaster/js-utils';
import { FormatColorFill, Gradient, InvertColors } from '@mui/icons-material';
import { FormatColorFill, Gradient, InvertColors, Undo } from '@mui/icons-material';
import { Button, Stack, Typography } from '@mui/material';
import { throttle } from 'lodash';
import React, { RefObject, useContext, useEffect, useMemo, useRef, useState } from 'react';
import { useStore } from 'zustand';
import { createLogger } from 'browser-bunyan';
import { SAVE_TIME } from '../../config.js';
import { ConfigContext, StateContext } from '../../state.js';
import { imageFromBlob } from '../../utils.js';
import { NumericField } from './NumericField';
export const DRAW_TIME = 25;
export const FULL_CIRCLE = 2 * Math.PI;
export const FULL_OPACITY = 1.0;
export const MASK_OPACITY = 0.75;
@ -45,17 +48,27 @@ export interface MaskCanvasProps {
onSave: (blob: Blob) => void;
}
const logger = createLogger({ name: 'react', level: 'debug' }); // TODO: hackeroni and cheese
export function MaskCanvas(props: MaskCanvasProps) {
const { source, mask } = props;
const { params } = mustExist(useContext(ConfigContext));
function drawBuffer() {
if (doesExist(brushRef.current) && doesExist(bufferRef.current) && doesExist(canvasRef.current)) {
const { ctx } = getClearContext(canvasRef);
ctx.globalAlpha = MASK_OPACITY;
ctx.drawImage(bufferRef.current, 0, 0);
function composite() {
if (doesExist(visibleRef.current)) {
const { ctx } = getClearContext(visibleRef);
if (maskState.current !== MASK_STATE.painting) {
if (doesExist(maskRef.current)) {
ctx.globalAlpha = MASK_OPACITY;
ctx.drawImage(maskRef.current, 0, 0);
}
if (doesExist(bufferRef.current)) {
ctx.globalAlpha = MASK_OPACITY;
ctx.drawImage(bufferRef.current, 0, 0);
}
if (doesExist(brushRef.current) && maskState.current !== MASK_STATE.painting) {
ctx.drawImage(brushRef.current, 0, 0);
}
}
@ -70,73 +83,94 @@ export function MaskCanvas(props: MaskCanvasProps) {
y: point.y,
}, brush.size);
drawBuffer();
composite();
}
function drawClicks(): void {
if (clicks.length > 0) {
function drawClicks(c2: Array<Point>, set: (value: React.SetStateAction<Array<Point>>) => void): boolean {
if (c2.length > 0) {
logger.debug('drawing clicks', { count: c2.length });
const { ctx } = getContext(bufferRef);
ctx.fillStyle = grayToRGB(brush.color, brush.strength);
for (const click of clicks) {
for (const click of c2) {
drawCircle(ctx, click, brush.size);
}
clicks.length = 0;
drawBuffer();
composite();
set([]);
return true;
}
return false;
}
function drawSource(file: Blob): void {
const image = new Image();
image.onload = () => {
const { canvas, ctx } = getClearContext(bufferRef);
ctx.globalAlpha = FULL_OPACITY;
ctx.drawImage(image, 0, 0, canvas.width, canvas.height);
async function drawMask(file: Blob): Promise<void> {
const image = await imageFromBlob(file);
logger.debug('draw mask');
URL.revokeObjectURL(src);
const { canvas, ctx } = getClearContext(maskRef);
ctx.globalAlpha = FULL_OPACITY;
ctx.drawImage(image, 0, 0, canvas.width, canvas.height);
drawBuffer();
};
const src = URL.createObjectURL(file);
image.src = src;
// getClearContext(bufferRef);
composite();
}
function finishPainting() {
logger.debug('finish painting');
if (doesExist(brushRef.current)) {
getClearContext(brushRef);
}
drawClicks();
if (drawClicks(clicks, setClicks) === false) {
logger.debug('force compositing');
composite();
}
if (maskState.current === MASK_STATE.painting) {
maskState.current = MASK_STATE.dirty;
}
}
function flushBuffer(): void {
if (doesExist(maskRef.current) && doesExist(bufferRef.current)) {
logger.debug('flush buffer');
const { ctx } = getContext(maskRef);
ctx.drawImage(bufferRef.current, 0, 0);
getClearContext(bufferRef);
composite();
}
}
function saveMask(): void {
if (doesExist(bufferRef.current)) {
if (doesExist(maskRef.current)) {
logger.debug('save mask');
if (maskState.current === MASK_STATE.clean) {
return;
}
bufferRef.current.toBlob((blob) => {
maskRef.current.toBlob((blob) => {
maskState.current = MASK_STATE.clean;
props.onSave(mustExist(blob));
});
}
}
const save = useMemo(() => throttle(saveMask, SAVE_TIME), []);
const draw = useMemo(() => throttle(drawClicks, DRAW_TIME), []);
const save = useMemo(() => throttle(saveMask, SAVE_TIME, {
trailing: true,
}), []);
// eslint-disable-next-line no-null/no-null
const brushRef = useRef<HTMLCanvasElement>(null);
// eslint-disable-next-line no-null/no-null
const bufferRef = useRef<HTMLCanvasElement>(null);
// eslint-disable-next-line no-null/no-null
const canvasRef = useRef<HTMLCanvasElement>(null);
const maskRef = useRef<HTMLCanvasElement>(null);
// eslint-disable-next-line no-null/no-null
const visibleRef = useRef<HTMLCanvasElement>(null);
// painting state
const maskState = useRef(MASK_STATE.clean);
@ -152,11 +186,17 @@ export function MaskCanvas(props: MaskCanvasProps) {
if (maskState.current === MASK_STATE.dirty) {
save();
}
return () => {
logger.debug('save cleanup');
};
}, [maskState.current]);
useEffect(() => {
if (doesExist(bufferRef.current) && doesExist(mask)) {
drawSource(mask);
drawMask(mask).catch((err) => {
// TODO: handle
});
}
}, [mask]);
@ -177,7 +217,9 @@ export function MaskCanvas(props: MaskCanvasProps) {
}, [source]);
// last resort to draw lost clicks
drawClicks();
// const lostClicks = drawClicks();
logger.debug('rendered', { clicks: clicks.length });
draw(clicks, setClicks);
const styles: React.CSSProperties = {
backgroundPosition: 'top left',
@ -210,12 +252,21 @@ export function MaskCanvas(props: MaskCanvasProps) {
}}
/>
<canvas
ref={canvasRef}
ref={maskRef}
height={params.height.default}
width={params.width.default}
style={{
display: 'none',
}}
/>
<canvas
ref={visibleRef}
height={params.height.default}
width={params.width.default}
style={styles}
onClick={(event) => {
const canvas = mustExist(canvasRef.current);
logger.debug('mouse click', { state: maskState.current, clicks: clicks.length });
const canvas = mustExist(visibleRef.current);
const bounds = canvas.getBoundingClientRect();
setClicks([...clicks, {
@ -223,17 +274,20 @@ export function MaskCanvas(props: MaskCanvasProps) {
y: event.clientY - bounds.top,
}]);
drawClicks();
drawClicks(clicks, setClicks);
maskState.current = MASK_STATE.dirty;
}}
onMouseDown={() => {
logger.debug('mouse down', { state: maskState.current, clicks: clicks.length });
maskState.current = MASK_STATE.painting;
flushBuffer();
}}
onMouseLeave={finishPainting}
onMouseOut={finishPainting}
onMouseUp={finishPainting}
onMouseMove={(event) => {
const canvas = mustExist(canvasRef.current);
const canvas = mustExist(visibleRef.current);
const bounds = canvas.getBoundingClientRect();
if (maskState.current === MASK_STATE.painting) {
@ -253,88 +307,100 @@ export function MaskCanvas(props: MaskCanvasProps) {
Black pixels in the mask will stay the same, white pixels will be replaced. The masked pixels will be blended
with the noise source before the diffusion model runs, giving it more variety to use.
</Typography>
<Stack direction='row' spacing={4}>
<NumericField
label='Brush Color'
min={COLORS.black}
max={COLORS.white}
step={1}
value={brush.color}
onChange={(color) => {
setBrush({ color });
}}
/>
<NumericField
label='Brush Size'
min={1}
max={64}
step={1}
value={brush.size}
onChange={(size) => {
setBrush({ size });
}}
/>
<NumericField
decimal
label='Brush Strength'
min={0}
max={1}
step={0.01}
value={brush.strength}
onChange={(strength) => {
setBrush({ strength });
}}
/>
<Button
variant='outlined'
startIcon={<FormatColorFill />}
onClick={() => {
floodCanvas(bufferRef, floodBlack);
drawBuffer();
maskState.current = MASK_STATE.dirty;
}}>
Fill with black
</Button>
<Button
variant='outlined'
startIcon={<FormatColorFill />}
onClick={() => {
floodCanvas(bufferRef, floodWhite);
drawBuffer();
maskState.current = MASK_STATE.dirty;
}}>
Fill with white
</Button>
<Button
variant='outlined'
startIcon={<InvertColors />}
onClick={() => {
floodCanvas(bufferRef, floodInvert);
drawBuffer();
maskState.current = MASK_STATE.dirty;
}}>
Invert
</Button>
<Button
variant='outlined'
startIcon={<Gradient />}
onClick={() => {
floodCanvas(bufferRef, floodBelow);
drawBuffer();
maskState.current = MASK_STATE.dirty;
}}>
Gray to black
</Button>
<Button
variant='outlined'
startIcon={<Gradient />}
onClick={() => {
floodCanvas(bufferRef, floodAbove);
drawBuffer();
maskState.current = MASK_STATE.dirty;
}}>
Gray to white
</Button>
<Stack>
<Stack direction='row' spacing={4}>
<NumericField
label='Brush Color'
min={COLORS.black}
max={COLORS.white}
step={1}
value={brush.color}
onChange={(color) => {
setBrush({ color });
}}
/>
<NumericField
label='Brush Size'
min={1}
max={64}
step={1}
value={brush.size}
onChange={(size) => {
setBrush({ size });
}}
/>
<NumericField
decimal
label='Brush Strength'
min={0}
max={1}
step={0.01}
value={brush.strength}
onChange={(strength) => {
setBrush({ strength });
}}
/>
</Stack>
<Stack direction='row' spacing={2}>
<Button
variant='outlined'
startIcon={<Undo />}
onClick={() => {
getClearContext(bufferRef);
composite();
}}
/>
<Button
variant='outlined'
startIcon={<FormatColorFill />}
onClick={() => {
floodCanvas(maskRef, floodBlack);
composite();
maskState.current = MASK_STATE.dirty;
}}>
Fill with black
</Button>
<Button
variant='outlined'
startIcon={<FormatColorFill />}
onClick={() => {
floodCanvas(maskRef, floodWhite);
composite();
maskState.current = MASK_STATE.dirty;
}}>
Fill with white
</Button>
<Button
variant='outlined'
startIcon={<InvertColors />}
onClick={() => {
floodCanvas(maskRef, floodInvert);
composite();
maskState.current = MASK_STATE.dirty;
}}>
Invert
</Button>
<Button
variant='outlined'
startIcon={<Gradient />}
onClick={() => {
floodCanvas(maskRef, floodBelow);
composite();
maskState.current = MASK_STATE.dirty;
}}>
Gray to black
</Button>
<Button
variant='outlined'
startIcon={<Gradient />}
onClick={() => {
floodCanvas(maskRef, floodAbove);
composite();
maskState.current = MASK_STATE.dirty;
}}>
Gray to white
</Button>
</Stack>
</Stack>
</Stack>;
}

5
gui/src/main.tsx
View File

@ -80,6 +80,11 @@ export async function main() {
...s.upscaleTab,
source: undefined,
},
blend: {
...s.blend,
mask: undefined,
sources: [],
}
};
},
storage: createJSONStorage(() => localStorage),

12
gui/src/utils.ts Normal file
View File

@ -0,0 +1,12 @@
export function imageFromBlob(blob: Blob): Promise<HTMLImageElement> {
return new Promise((res, rej) => {
const image = new Image();
image.onload = () => {
URL.revokeObjectURL(src);
res(image);
};
const src = URL.createObjectURL(blob);
image.src = src;
});
}

36
gui/yarn.lock
View File

@ -80,6 +80,32 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@browser-bunyan/console-formatted-stream@^1.8.0":
version "1.8.0"
resolved "https://registry.yarnpkg.com/@browser-bunyan/console-formatted-stream/-/console-formatted-stream-1.8.0.tgz#dda9dcab6ce445cbf2911045709930757e5d48c1"
integrity sha512-Lg5SC2uXrvZ6aLwLZT6SErfN1Is4NcrTOb5km4BW/BfL8Lv0CfpsYuhuD7ltdURL6awTYBUiT+BwhKw1Xd9glQ==
dependencies:
"@browser-bunyan/levels" "^1.8.0"
"@browser-bunyan/console-plain-stream@^1.8.0":
version "1.8.0"
resolved "https://registry.yarnpkg.com/@browser-bunyan/console-plain-stream/-/console-plain-stream-1.8.0.tgz#18cd8fe879a0f576cf84c4fa4647e86cd3feea3e"
integrity sha512-S0WNsH5zvMfkbayIx90wANGHQ8l3Bvd7mjgy95/bYmUzcI+Mwkv2eJcSufdTP/MbdHBhjv/lEdLDOXEPBi+w3A==
dependencies:
"@browser-bunyan/levels" "^1.8.0"
"@browser-bunyan/console-raw-stream@^1.8.0":
version "1.8.0"
resolved "https://registry.yarnpkg.com/@browser-bunyan/console-raw-stream/-/console-raw-stream-1.8.0.tgz#5d0438139bbffd9ed779241df6ae7e5f3a2a7b0c"
integrity sha512-6M/xEiNckbFslQMaS1BHAxvuvN1Wtbh/aq4UzQD3fjEPFCxtubvf4KyzwPxUXA5CXq7leVZ+cibEUCRBsm5bzg==
dependencies:
"@browser-bunyan/levels" "^1.8.0"
"@browser-bunyan/levels@^1.8.0":
version "1.8.0"
resolved "https://registry.yarnpkg.com/@browser-bunyan/levels/-/levels-1.8.0.tgz#1c0a98d04284e0620e8ee414d7ce43385080a5cf"
integrity sha512-f9oSDik8kAl+4rhVyHqIr012P1boHFUKc7D9nzA5+lDsFoP90UQnDwpseqBdF2mTaWYju10E7h+GdH8u+7MHOQ==
"@emotion/babel-plugin@^11.10.5":
version "11.10.5"
resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.10.5.tgz#65fa6e1790ddc9e23cc22658a4c5dea423c55c3c"
@ -879,6 +905,16 @@ broadcast-channel@^3.4.1:
rimraf "3.0.2"
unload "2.2.0"
browser-bunyan@^1.8.0:
version "1.8.0"
resolved "https://registry.yarnpkg.com/browser-bunyan/-/browser-bunyan-1.8.0.tgz#6b9662fea571c642fce80ad002d62e3ea1453393"
integrity sha512-Et1TaRUm8m2oy4OTi69g0qAM8wqpofACUgkdBnj1Kq2aC8Wpl8w+lNevebPG6zKH2w0Aq+BHiAXWwjm0/QbkaQ==
dependencies:
"@browser-bunyan/console-formatted-stream" "^1.8.0"
"@browser-bunyan/console-plain-stream" "^1.8.0"
"@browser-bunyan/console-raw-stream" "^1.8.0"
"@browser-bunyan/levels" "^1.8.0"
browser-stdout@1.3.1:
version "1.3.1"
resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60"