1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

# SPDX-License-Identifier: MIT
# SPDX-FileCopyrightText: Copyright 2024 Sam Blenny
#
# WebSerialDisplay: A virtual CircuitPython display over Web Serial
#
# Related Docs and Code:
# - https://learn.adafruit.com/adafruit-picowbell-camera-breakout
# - https://docs.circuitpython.org/projects/ov5640/en/latest/
# - https://github.com/adafruit/Adafruit_CircuitPython_OV5640/blob/main/adafruit_ov5640/__init__.py
# - https://docs.circuitpython.org/en/latest/docs/library/binascii.html
#
from binascii import b2a_base64
from board import (
    GP0, GP2, GP3, GP4, GP5, GP6, GP7, GP8, GP9, GP10,
    GP11, GP12, GP13, GP18, GP19, GP22,
)
from busio import I2C
from digitalio import DigitalInOut
from gc import collect, mem_free
from sys import stdout
from time import sleep

import adafruit_bus_device
from adafruit_ov5640 import (
    OV5640, OV5640_SIZE_96X96, OV5640_SIZE_240X240, OV5640_COLOR_GRAYSCALE
)

def gcCol():
    # Collect garbage and print free memory
    collect()
    print("mem_free", mem_free())

def send(buf):
    # Encode the frame buffer (buf) as base64 and send it over the serial port.
    # CAUTION: This assumes len(buf) is evenly divisible by 96, which is
    # true for 96x96 and 240x240 frames.
    # Performance Notes: Caching function references as local vars is a
    # MicroPython speedup trick that avoids repeated dictionary lookups. Also,
    # using sys.stdout.write() here is *way* faster than using print().
    wr = stdout.write
    b64 = b2a_base64
    wr(b'-----BEGIN FRAME-----\n')
    stride = 96
    for i in range(0, len(buf), stride):
        wr(b64(buf[i:i+stride]))
    wr('-----END FRAME-----\n')

def main():
    # Make a buffer to hold captured pixel data
    gcCol()
    size = 240
    buf = bytearray(size * size)
    # Configure the camera
    cam = OV5640(
        I2C(GP5, GP4),
        vsync=GP0,
        href=GP2,
        clock=GP3,
        data_pins=(GP6, GP7, GP8, GP9, GP10, GP11, GP12, GP13),
        mclk=None,
        size=(OV5640_SIZE_96X96 if (size==96) else OV5640_SIZE_240X240),
    )
    cam.flip_y = True
    cam.colorspace = OV5640_COLOR_GRAYSCALE
    gcCol()
    # Capture and send camera frames at approximately 2 frames per second
    while True:
        sleep(0.15)  # 0.15 -> 2fps @ 240x240px
        cam.capture(buf)
        send(buf)
        gcCol()

main()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

<!DOCTYPE html><html lang="en"><head><meta charset="utf-8" />
<!-- SPDX-License-Identifier: MIT -->
<!-- SPDX-FileCopyrightText: Copyright 2024 Sam Blenny -->
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>WebSerialDisplay</title>
<style>
 :root{--F: #ffdcec; --B: #1d1d1d; --H: #25c3c0; --L: #459cda; --O: #444;
       --BTN: #32ea39; --PAD: 25px; --SIZE: 480px; }
 @media (prefers-color-scheme: light){
  :root{--F: #101010; --B: #f5f5f5; --H: #313131; --L: #3f40f9; --O: #aaa;
        --BTN: #3dff4d; }
 }
 html{ background: var(--B); -webkit-text-size-adjust: none; }
 body{ background: var(--B); color: var(--F); min-height: 100vh;
  width: 100%; font-family: sans-serif; font-size: 17px; }
 html,body,h1,h2,main,input{ margin: 0; padding: 0; box-sizing: border-box; }
 video,canvas{ margin: 0; padding: 0; box-sizing: content-box; }
 p,li,label{ line-height: 1.23em; }
 label,select{ display: inline-block; min-height: 20px; }
 label{ margin: 12px 3px 12px 18px; }
 input{ margin: 12px 3px 12px 3px; }
 h1{ margin: 10px 0 10px 0px; font-size: 24px; color: var(--H); }
 h2{ margin: 20px 0 10px 0px; font-size: 20px; color: var(--H); }
 a{ color: var(--L); }
 p{ margin: 11px 0; }
 .clear{ clear: both; }
 main{ margin: 0 auto; padding: 2px var(--PAD) 115px var(--PAD);
  max-width: calc((2 * var(--PAD)) + var(--SIZE)); }
 #serial{ float: right; background: var(--BTN); font-size: 1.1em;
  padding: 10px; border-radius: 13px; border: 2px solid var(--H); }
 #serial.on{ background: var(--B); color: var(--F); }
 canvas{ display: inline-block; width: var(--SIZE); height: var(--SIZE);
  margin: 10px auto; background: var(--O); }
 @media (max-width: 530px) { /* try not to overflow narrow screens */
  :root{--PAD: 13px;}
  body{ font-size: 16px; }
  canvas{ width: 100%; height: auto; }
 }
</style>
</head>
<body>
<main>
<h1>WebSerialDisplay</h1>

<button id="serial">Connect</button>

<p>CircuitPython web serial virtual display<br>
status: <span id="status">loading...</span></p>

<p class="clear">
<canvas id="canvas">
</p>

<script src="main.js"></script>

</main>
</body>
</html>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203

// SPDX-License-Identifier: MIT
// SPDX-FileCopyrightText: Copyright 2024 Sam Blenny
//
// WebSerialDisplay: A virtual CircuitPython display over Web Serial
//
// Related Docs:
// - https://developer.chrome.com/docs/capabilities/serial
// - https://codelabs.developers.google.com/codelabs/web-serial#0
// - https://developer.mozilla.org/en-US/docs/Web/API/Web_Serial_API
// - https://developer.mozilla.org/en-US/docs/Web/API/SerialPort
// - https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Concepts
// - https://developer.mozilla.org/en-US/docs/Web/API/TextDecoderStream
// - https://developer.mozilla.org/en-US/docs/Glossary/Base64
//
"use strict";

const STATUS = document.querySelector('#status');   // Status span
const SER_BTN = document.querySelector('#serial');  // Start Serial button
const CANVAS = document.querySelector('#canvas');   // Canvas

const CTX = CANVAS.getContext("2d", {willReadFrequently: true});

// Serial Port
var SER_PORT = null;

// Update status line span
function setStatus(s) {
    STATUS.textContent = s;
}

// Exapand pixel values from the luma array into the RGBA array as grayscale
function expandIntoRGBA(luma, rgba) {
    // luma is Uint8ClampedArray using 1 byte per pixel
    // rgba is Uing8ClampedArray using 4 bytes per pixel
    let i = 0;
    for (const Y of luma) {
        rgba[i] = Y;
        rgba[i+1] = Y;
        rgba[i+2] = Y;
        rgba[i+3] = 255;
        i += 4;
    }
}

// Disconnect serial port and stop updating the canvas
async function disconnect(status) {
    if (SER_PORT) {
        await SER_PORT.forget();
        SER_PORT = null;
    }
    SER_BTN.classList.remove('on');
    SER_BTN.textContent = 'Connect';
    setStatus(status ? status : 'disconnected');
}

// Update HTML canvas element with pixels for a new virtual display frame
async function paintFrame(data) {
    // Set size of virtual display (should be 96x96 or 240x240)
    const w = Math.round(Math.sqrt(data.length));
    const h = w;
    CANVAS.width = w;
    CANVAS.height = h;
    // getImageData returns RGBA Uint8ClampedArray of pixels in row-major order
    const imageData = CTX.getImageData(0, 0, w, h);
    const rgba = imageData.data;
    const luma = Uint8ClampedArray.from(data);
    expandIntoRGBA(luma, rgba);
    CTX.putImageData(imageData, 0, 0);
}

// Parse complete lines to assemble frames
async function parseLine(line, state) {
    if(!state.frameSync) {
        // Ignore lines until the first start of frame marker
        // Wait to sync with start of frame
        if (line == '-----BEGIN FRAME-----') {
            state.frameSync = true;
            state.data = [];
        } else if (line.startsWith('mem_free ')) {
            // Only log mem_free lines when the number has changed
            if (line != state.memFree) {
                state.memFree = line;
                console.log(line);
            }
        }
    } else {
        // When frame sync is locked, save base64 data until end of frame mark
        if (line == '-----END FRAME-----') {
            state.frameSync = false;
            try {
                // Decode the base64 using the Data URL decoder because the
                // old school btoa() decoder function is problematic
                const dataUrlPrefix = 'data:application/octet-stream;base64,';
                const buf = await fetch(dataUrlPrefix + state.data.join(''));
                const data = new Uint8Array(await buf.arrayBuffer());
                paintFrame(data);
            } catch (e) {
                console.log("bad frame", e);
            }
        } else {
            // This is a base64 data chunk
            state.data.push(line);
        }
    }
}

// Parse a chunk of serial data to assemble complete lines.
// CAUTION: This expects '\r\n' line endings!
async function parseChunk(chunk, state) {
    if (!state.lineSync) {
        // Ignore everything up to the first line ending, then start buffering
        // the next line
        const n = chunk.indexOf('\r\n');
        if (n >= 0) {
            state.lineBuf = (chunk.slice(n+2));
            state.lineSync = true;
        }
    } else {
        // Once line sync is locked, just append the next chunk
        state.lineBuf += chunk;
    }
    // Parse complete lines off the front of the buffered chunks
    var i = state.lineBuf.indexOf('\r\n');
    while(i >= 0) {
        const line = state.lineBuf.substr(0, i);
        state.lineBuf = state.lineBuf.substr(i+2);
        parseLine(line, state);
        i = state.lineBuf.indexOf('\r\n');
    }
}

// Decode base64 encoded frame buffer updates from the serial port
async function readFrames(port) {
    const reader = port.readable
        .pipeThrough(new TextDecoderStream())
        .getReader();
    const state = {
        lineSync: false,
        frameSync: false,
        lineBuf: '',
        data: [],
        memFree: '',
    };
    while(port.readable) {
        try {
            const {done, value} = await reader.read();
            if (done) {
                reader.releaseLock();
                break;
            }
            if (value.length > 0) {
                parseChunk(value, state);
            }
        } catch(err) {
            // This is normal for a disconnect (button or USB cable)
            break;
        }
    }
}

// Attempt to start virtual display with data feed over Web Serial
function connect() {
    if (!('serial' in navigator)) {
        setStatus('Browser does not support Web Serial');
        alert('This browser does not support Web Serial');
        return;
    }
    // Define a filter for Adafruit's USB vendor ID (works for Pi Pico)
    const circuitpyFilter = [{usbVendorId: 0x239a}];
    // Request access to serial port (trigger a browser permission prompt)
    navigator.serial
    .requestPort({filters: circuitpyFilter})
    .then(async (response) => {
        SER_PORT = await response;
        SER_PORT.ondisconnect = async (event) => {
            await event.target.close();
            disconnect('serial device unplugged');
        };
        await SER_PORT.open({baudRate: 115200});
        // Update HTML button
        SER_BTN.classList.add('on');
        SER_BTN.textContent = 'disconnect';
        // Update status line
        setStatus('connected');
        // Begin reading frame buffer updates
        readFrames(SER_PORT);
    })
    .catch((err) => {
        SER_PORT = null;
        setStatus('no serial port selected');
    });
}

// Add on/off event handlers to the button
SER_BTN.addEventListener('click', function() {
    if(SER_BTN.classList.contains('on')) {
        disconnect();
    } else {
        connect();
    }
});

setStatus("ready");