apextoaster
/
onnx-web
1
0
Fork 0
Code Actions

feat(gui): show source behind mask with offscreen painting

This commit is contained in:
Sean Sube 2023-01-13 23:38:43 -06:00
parent f46647cf6d
commit e915ab5b8d
2 changed files with 85 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
gui/src/components/Inpaint.tsx
View File

@ -47,15 +47,6 @@ export function Inpaint(props: InpaintProps) {
onSuccess: () => query.invalidateQueries({ queryKey: 'ready' }), onSuccess: () => query.invalidateQueries({ queryKey: 'ready' }),
}); });
useEffect(function changeSource() {
// draw the source to the canvas if the mask has not been set
if (doesExist(params.source) && doesExist(params.mask) === false) {
setInpaint({
mask: params.source,
});
}
}, [params.source]);
return <Box> return <Box>
<Stack spacing={2}> <Stack spacing={2}>
<ImageInput <ImageInput
@ -78,11 +69,16 @@ export function Inpaint(props: InpaintProps) {
}); });
}} }}
renderImage={(image) => renderImage={(image) =>
<MaskCanvas config={config} source={image} onSave={(mask) => { <MaskCanvas
setInpaint({ config={config}
mask, base={params.source}
}); source={image}
}} /> onSave={(mask) => {
setInpaint({
mask,
});
}}
/>
} }
/> />
<ImageControl <ImageControl

105
gui/src/components/MaskCanvas.tsx
View File

@ -2,12 +2,13 @@ import { doesExist, Maybe, mustExist } from '@apextoaster/js-utils';
import { FormatColorFill, Gradient } from '@mui/icons-material'; import { FormatColorFill, Gradient } from '@mui/icons-material';
import { Button, Stack } from '@mui/material'; import { Button, Stack } from '@mui/material';
import { throttle } from 'lodash'; import { throttle } from 'lodash';
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import React, { useEffect, useMemo, useRef, useState } from 'react';
import { ConfigParams, DEFAULT_BRUSH, SAVE_TIME } from '../config.js'; import { ConfigParams, DEFAULT_BRUSH, SAVE_TIME } from '../config.js';
import { NumericField } from './NumericField'; import { NumericField } from './NumericField';
export const FULL_CIRCLE = 2 * Math.PI; export const FULL_CIRCLE = 2 * Math.PI;
export const MASK_OPACITY = 0.75;
export const PIXEL_SIZE = 4; export const PIXEL_SIZE = 4;
export const PIXEL_WEIGHT = 3; export const PIXEL_WEIGHT = 3;
@ -61,23 +62,24 @@ export interface Point {
export interface MaskCanvasProps { export interface MaskCanvasProps {
config: ConfigParams; config: ConfigParams;
base?: Maybe<Blob>;
source?: Maybe<Blob>; source?: Maybe<Blob>;
onSave: (blob: Blob) => void; onSave: (blob: Blob) => void;
} }
export function MaskCanvas(props: MaskCanvasProps) { export function MaskCanvas(props: MaskCanvasProps) {
const { config, source } = props; const { base, config, source } = props;
function floodMask(flood: FloodFn) { function floodMask(flood: FloodFn) {
const canvas = mustExist(canvasRef.current); const buffer = mustExist(bufferRef.current);
const ctx = mustExist(canvas.getContext('2d')); const ctx = mustExist(buffer.getContext('2d'));
const image = ctx.getImageData(0, 0, canvas.width, canvas.height); const image = ctx.getImageData(0, 0, buffer.width, buffer.height);
const pixels = image.data; const pixels = image.data;
for (let x = 0; x < canvas.width; ++x) { for (let x = 0; x < buffer.width; ++x) {
for (let y = 0; y < canvas.height; ++y) { for (let y = 0; y < buffer.height; ++y) {
const i = (y * canvas.width * PIXEL_SIZE) + (x * PIXEL_SIZE); const i = (y * buffer.width * PIXEL_SIZE) + (x * PIXEL_SIZE);
const hue = (pixels[i] + pixels[i + 1] + pixels[i + 2]) / PIXEL_WEIGHT; const hue = (pixels[i] + pixels[i + 1] + pixels[i + 2]) / PIXEL_WEIGHT;
const final = flood(hue); const final = flood(hue);
@ -92,18 +94,29 @@ export function MaskCanvas(props: MaskCanvasProps) {
} }
function saveMask(): void { function saveMask(): void {
if (doesExist(canvasRef.current)) { if (doesExist(bufferRef.current)) {
if (state.current === MASK_STATE.clean) { if (maskState.current === MASK_STATE.clean) {
return; return;
} }
canvasRef.current.toBlob((blob) => { bufferRef.current.toBlob((blob) => {
state.current = MASK_STATE.clean; maskState.current = MASK_STATE.clean;
props.onSave(mustExist(blob)); props.onSave(mustExist(blob));
}); });
} }
} }
function drawBuffer() {
if (doesExist(bufferRef.current) && doesExist(canvasRef.current)) {
const dest = mustExist(canvasRef.current);
const ctx = mustExist(dest.getContext('2d'));
ctx.clearRect(0, 0, dest.width, dest.height);
ctx.globalAlpha = MASK_OPACITY;
ctx.drawImage(bufferRef.current, 0, 0);
}
}
function drawCircle(ctx: CanvasRenderingContext2D, point: Point): void { function drawCircle(ctx: CanvasRenderingContext2D, point: Point): void {
ctx.beginPath(); ctx.beginPath();
ctx.arc(point.x, point.y, brushSize, 0, FULL_CIRCLE); ctx.arc(point.x, point.y, brushSize, 0, FULL_CIRCLE);
@ -113,10 +126,12 @@ export function MaskCanvas(props: MaskCanvasProps) {
function drawSource(file: Blob): void { function drawSource(file: Blob): void {
const image = new Image(); const image = new Image();
image.onload = () => { image.onload = () => {
const canvas = mustExist(canvasRef.current); const buffer = mustExist(bufferRef.current);
const ctx = mustExist(canvas.getContext('2d')); const ctx = mustExist(buffer.getContext('2d'));
ctx.drawImage(image, 0, 0); ctx.drawImage(image, 0, 0);
URL.revokeObjectURL(src); URL.revokeObjectURL(src);
drawBuffer();
}; };
const src = URL.createObjectURL(file); const src = URL.createObjectURL(file);
@ -124,27 +139,30 @@ export function MaskCanvas(props: MaskCanvasProps) {
} }
function finishPainting() { function finishPainting() {
if (state.current === MASK_STATE.painting) { if (maskState.current === MASK_STATE.painting) {
state.current = MASK_STATE.dirty; maskState.current = MASK_STATE.dirty;
save(); save();
} }
} }
const save = useMemo(() => throttle(saveMask, SAVE_TIME), []); const save = useMemo(() => throttle(saveMask, SAVE_TIME), []);
// eslint-disable-next-line no-null/no-null
const bufferRef = useRef<HTMLCanvasElement>(null);
// eslint-disable-next-line no-null/no-null // eslint-disable-next-line no-null/no-null
const canvasRef = useRef<HTMLCanvasElement>(null); const canvasRef = useRef<HTMLCanvasElement>(null);
// painting state // painting state
const state = useRef(MASK_STATE.clean); const maskState = useRef(MASK_STATE.clean);
const [background, setBackground] = useState<string>();
const [clicks, setClicks] = useState<Array<Point>>([]); const [clicks, setClicks] = useState<Array<Point>>([]);
const [brushColor, setBrushColor] = useState(DEFAULT_BRUSH.color); const [brushColor, setBrushColor] = useState(DEFAULT_BRUSH.color);
const [brushSize, setBrushSize] = useState(DEFAULT_BRUSH.size); const [brushSize, setBrushSize] = useState(DEFAULT_BRUSH.size);
useEffect(() => { useEffect(() => {
// including clicks.length prevents the initial render from saving a blank canvas // including clicks.length prevents the initial render from saving a blank canvas
if (doesExist(canvasRef.current) && state.current === MASK_STATE.painting && clicks.length > 0) { if (doesExist(bufferRef.current) && maskState.current === MASK_STATE.painting && clicks.length > 0) {
const ctx = mustExist(canvasRef.current.getContext('2d')); const ctx = mustExist(bufferRef.current.getContext('2d'));
ctx.fillStyle = grayToRGB(brushColor); ctx.fillStyle = grayToRGB(brushColor);
for (const click of clicks) { for (const click of clicks) {
@ -152,34 +170,61 @@ export function MaskCanvas(props: MaskCanvasProps) {
} }
clicks.length = 0; clicks.length = 0;
drawBuffer();
} }
}, [clicks.length]); }, [clicks.length]);
useEffect(() => { useEffect(() => {
if (state.current === MASK_STATE.dirty) { if (maskState.current === MASK_STATE.dirty) {
save(); save();
} }
}, [state.current]); }, [maskState.current]);
useEffect(() => { useEffect(() => {
if (doesExist(canvasRef.current) && doesExist(source)) { if (doesExist(bufferRef.current) && doesExist(source)) {
drawSource(source); drawSource(source);
} }
}, [source]); }, [source]);
useEffect(() => {
if (doesExist(base)) {
if (doesExist(background)) {
URL.revokeObjectURL(background);
}
setBackground(URL.createObjectURL(base));
}
}, [base]);
const styles: React.CSSProperties = {
maxHeight: config.height.default,
maxWidth: config.width.default,
};
if (doesExist(background)) {
styles.backgroundImage = `url(${background})`;
}
return <Stack spacing={2}> return <Stack spacing={2}>
<canvas
ref={bufferRef}
height={config.height.default}
width={config.width.default}
style={{
display: 'none',
}}
/>
<canvas <canvas
ref={canvasRef} ref={canvasRef}
height={config.height.default} height={config.height.default}
width={config.width.default} width={config.width.default}
style={{ style={styles}
maxHeight: config.height.default,
maxWidth: config.width.default,
}}
onClick={(event) => { onClick={(event) => {
const canvas = mustExist(canvasRef.current); const canvas = mustExist(canvasRef.current);
const bounds = canvas.getBoundingClientRect(); const bounds = canvas.getBoundingClientRect();
const ctx = mustExist(canvas.getContext('2d'));
const buffer = mustExist(bufferRef.current);
const ctx = mustExist(buffer.getContext('2d'));
ctx.fillStyle = grayToRGB(brushColor); ctx.fillStyle = grayToRGB(brushColor);
drawCircle(ctx, { drawCircle(ctx, {
@ -187,17 +232,17 @@ export function MaskCanvas(props: MaskCanvasProps) {
y: event.clientY - bounds.top, y: event.clientY - bounds.top,
}); });
state.current = MASK_STATE.dirty; maskState.current = MASK_STATE.dirty;
save(); save();
}} }}
onMouseDown={() => { onMouseDown={() => {
state.current = MASK_STATE.painting; maskState.current = MASK_STATE.painting;
}} }}
onMouseLeave={finishPainting} onMouseLeave={finishPainting}
onMouseOut={finishPainting} onMouseOut={finishPainting}
onMouseUp={finishPainting} onMouseUp={finishPainting}
onMouseMove={(event) => { onMouseMove={(event) => {
if (state.current === MASK_STATE.painting) { if (maskState.current === MASK_STATE.painting) {
const canvas = mustExist(canvasRef.current); const canvas = mustExist(canvasRef.current);
const bounds = canvas.getBoundingClientRect(); const bounds = canvas.getBoundingClientRect();