I am working on a Chrome extension that prevents images on a website from loading if they contain certain types of content.
I am having trouble passing the image in a format that allows it to be opened by the Python backend, which is supposed to analyze the image using a fastai model and return the prediction.
I have tried base64 encoding the image before passing it, passing it as a Form, as well as using a canvas, but nothing has worked.
Here is my latest attempt at the javascript function:
function DoElement() {
if (showAll)
return;
let el = this;
if (isImg(el)) {
//attach load event - needed 1) as we need to catch it after it is switched for the blankImg, 2) in case the img gets changed to something else later
DoLoadEventListener(el, true);
//see if not yet loaded
if (!el.complete) {
//hide, to avoid flash until load event is handled
MarkWizmaged(el, true);
DoHidden(el, true);
return;
}
let elWidth = el.width, elHeight = el.height;
if (el.src == blankImg && !el.srcset) { //was successfully replaced
DoHidden(el, false);
}
else if ((elWidth == 0 || elWidth > _settings.maxSafe) && (elHeight == 0 || elHeight > _settings.maxSafe)) { //needs to be hidden - we need to catch 0 too, as sometimes images start off as zero
DoMouseEventListeners(el, true);
if (!el.wzmHasTitleAndSizeSetup) {
el.style.width = elWidth + 'px';
el.style.height = elHeight + 'px';
if (!el.title)
if (el.alt)
el.title = el.alt;
else {
el.src.match(/([-\w]+)(\.[\w]+)?$/i);
el.title = RegExp.$1;
}
el.wzmHasTitleAndSizeSetup = true;
}
DoHidden(el, true);
DoImgSrc(el, true);
DoWizmageBG(el, true);
el.src = blankImg;
var canvas = document.createElement('canvas');
var context = canvas.getContext('2d');
canvas.width = el.width;
canvas.height = el.height;
context.drawImage(el, 0, 0);
var imageData = context.getImageData(0, 0, el.width, el.height);
// Create a Blob object from the image data
var blob = new Blob([imageData.data.buffer], { type: 'image/png' });
// Create a FormData object and append the blob to it
var formData = new FormData();
formData.append('image', blob, 'image.png');
// Send the request to the server
fetch('http://localhost:8000/predict', {
method: 'POST',
body: formData
})
.then((response) => response.json())
.then((data) => {
if (data.prediction === 'nudity') {
MarkWizmaged(el, true);
DoHidden(el, true);
} else {
MarkWizmaged(el, false);
DoHidden(el, false);
}
})
.catch((error) => {
console.error('Error:', error);
});
Here is my python backend, which I am hosting locally:
from fastai.vision.all import *
from starlette.applications import Starlette
from starlette.middleware.cors import CORSMiddleware
from starlette.responses import JSONResponse
import io
import base64
app = Starlette()
app.add_middleware(CORSMiddleware, allow_origins=['*'], allow_headers=['*'], allow_methods=['*'])
# Load the trained model
learn = load_learner('/Users/.../Documents/Personal/WorkSafe/Backend/resnet18.pkl')
# Define the prediction route
@app.route("/predict", methods=["POST"])
async def predict(request):
data = await request.form()
image = data.get("image", None)
if image is not None:
# Retrieve the file data from the form
image_data = await image.read()
# Reset the stream position to the beginning
image_data.seek(0)
img = Image.open(io.BytesIO(image_data))
img = img.convert('RGB')
# Make the prediction
prediction, _, _ = learn.predict(img)
return JSONResponse({'prediction': str(prediction)})
When I run this, it gives the following error:
Traceback (most recent call last):
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/uvicorn/protocols/http/h11_impl.py", line 428, in run_asgi
result = await app( # type: ignore[func-returns-value]
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
return await self.app(scope, receive, send)
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/applications.py", line 122, in __call__
await self.middleware_stack(scope, receive, send)
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/middleware/errors.py", line 184, in __call__
raise exc
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/middleware/errors.py", line 162, in __call__
await self.app(scope, receive, _send)
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/middleware/cors.py", line 91, in __call__
await self.simple_response(scope, receive, send, request_headers=headers)
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/middleware/cors.py", line 146, in simple_response
await self.app(scope, receive, send)
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/_exception_handler.py", line 57, in wrapped_app
raise exc
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/_exception_handler.py", line 46, in wrapped_app
await app(scope, receive, sender)
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/routing.py", line 727, in __call__
await route.handle(scope, receive, send)
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/routing.py", line 285, in handle
await self.app(scope, receive, send)
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/routing.py", line 74, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/_exception_handler.py", line 57, in wrapped_app
raise exc
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/_exception_handler.py", line 46, in wrapped_app
await app(scope, receive, sender)
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/starlette/routing.py", line 69, in app
response = await func(request)
File "/Users/shaulsaitowitz/Documents/Personal/WorkSafe/Backend/worksafe_app.py", line 27, in predict
image_data.seek(0)
I had just added .seek(0) at the suggestion of another answer to fix the following error, which I got previously:
response = await func(request)
File "/Users/shaulsaitowitz/Documents/Personal/WorkSafe/Backend/worksafe_app.py", line 26, in predict
img = Image.open(io.BytesIO(image_data))
File "/Users/shaulsaitowitz/opt/anaconda3/lib/python3.8/site-packages/PIL/Image.py", line 2967, in open
raise UnidentifiedImageError(
PIL.UnidentifiedImageError: cannot identify image file <_io.BytesIO object at 0x7fdff4b43ef0>
Can anyone tell me how to fix this? (I find it difficult to provide a Minimal Reproducible Example because running it would require recreating a Chrome extension for demonstration.)
These questions were suggested as similar but are actually not helpful for how to pass an image: passing an image from background page to content script javascript Message passing to chrome extension from local python
Edit: My question was marked as a duplicate of How to make a cross-origin request in a content script (currently blocked by CORB despite the correct CORS headers)?, however they are unrelated. That question pertains to not being able to receive a response, whereas I am asking how to post a request in a way that can be handled by the backend.
