Make SD cards available over web workflow

This changes storage.mount() to require that a mount point exist
on the parent file system.

A bug in background tasks is also fixed where the function
parameter is cleared on pending callbacks during "reset".

Disk usage is shown on the directory listing and changes based on
the mounted file system. Writable is also loaded per-directory.

Fixes micropython#8108. Fixes micropython#8690. Fixes micropython#8107.

docs/workflows.md

@@ -165,6 +165,13 @@ Returns a JSON representation of the directory.
 * `403 Forbidden` - No `CIRCUITPY_WEB_API_PASSWORD` set
 * `404 Not Found` - Missing directory
 
+Returns directory information:
+* `free`: Count of free blocks on the disk holding this directory.
+* `total`: Total blocks that make up the disk holding this directory.
+* `block_size`: Size of a block in bytes.
+* `writable`: True when CircuitPython and the web workflow can write to the disk. USB may claim a disk instead.
+* `files`: Array of objects. One for each file.
+
 Returns information about each file in the directory:
 
 * `name` - File name. No trailing `/` on directory names
@@ -179,14 +186,20 @@ curl -v -u :passw0rd -H "Accept: application/json" -L --location-trusted http://
 ```
 
 ```json
-[
-	{
-		"name": "world.txt",
-		"directory": false,
-		"modified_ns": 946934328000000000,
-		"file_size": 12
-	}
-]
+{
+	"free": 451623,
+	"total": 973344,
+	"block_size": 32768,
+	"writable": true,
+	"files": [
+		{
+			"name": "world.txt",
+			"directory": false,
+			"modified_ns": 946934328000000000,
+			"file_size": 12
+		}
+	]
+}
 ```
 
 ##### PUT
@@ -373,10 +386,10 @@ curl -v -L http://circuitpython.local/cp/devices.json
 Returns information about the attached disk(s). A list of objects, one per disk.
 
 * `root`: Filesystem path to the root of the disk.
-* `free`: Count of free bytes on the disk.
+* `free`: Count of free blocks on the disk.
+* `total`: Total blocks that make up the disk.
 * `block_size`: Size of a block in bytes.
 * `writable`: True when CircuitPython and the web workflow can write to the disk. USB may claim a disk instead.
-* `total`: Total bytes that make up the disk.
 
 Example:
 ```sh
@@ -405,7 +418,7 @@ This is an authenticated endpoint in both modes.
 
 Returns information about the device.
 
-* `web_api_version`: Between `1` and `3`. This versions the rest of the API and new versions may not be backwards compatible. See below for more info.
+* `web_api_version`: Between `1` and `4`. This versions the rest of the API and new versions may not be backwards compatible. See below for more info.
 * `version`: CircuitPython build version.
 * `build_date`: CircuitPython build date.
 * `board_name`: Human readable name of the board.
@@ -467,3 +480,5 @@ Only one WebSocket at a time is supported.
 * `1` - Initial version.
 * `2` - Added `/cp/diskinfo.json`.
 * `3` - Changed `/cp/diskinfo.json` to return a list in preparation for multi-disk support.
+* `4` - Changed directory json to an object with additional data. File list is under `files` and is
+  the same as the old format.

extmod/vfs.h

@@ -48,6 +48,8 @@
 #define MP_BLOCKDEV_FLAG_USB_WRITABLE             (0x0010)
 // Bit set when the above flag is checked before opening a file for write.
 #define MP_BLOCKDEV_FLAG_CONCURRENT_WRITE_PROTECTED (0x0020)
+// Bit set when something has claimed the right to mutate the blockdev.
+#define MP_BLOCKDEV_FLAG_LOCKED (0x0040)
 
 // constants for block protocol ioctl
 #define MP_BLOCKDEV_IOCTL_INIT          (1)

extmod/vfs_blockdev.c

@@ -30,12 +30,41 @@
 #include "py/mperrno.h"
 #include "extmod/vfs.h"
 
+#if CIRCUITPY_SDCARDIO
+#include "shared-bindings/sdcardio/SDCard.h"
+#endif
+#if CIRCUITPY_SDIOIO
+#include "shared-bindings/sdioio/SDCard.h"
+#endif
+
+
 #if MICROPY_VFS
 
 void mp_vfs_blockdev_init(mp_vfs_blockdev_t *self, mp_obj_t bdev) {
     mp_load_method(bdev, MP_QSTR_readblocks, self->readblocks);
     mp_load_method_maybe(bdev, MP_QSTR_writeblocks, self->writeblocks);
     mp_load_method_maybe(bdev, MP_QSTR_ioctl, self->u.ioctl);
+
+    // CIRCUITPY-CHANGE: Support native SD cards.
+    #if CIRCUITPY_SDCARDIO
+    if (mp_obj_get_type(bdev) == &sdcardio_SDCard_type) {
+        self->flags |= MP_BLOCKDEV_FLAG_NATIVE | MP_BLOCKDEV_FLAG_HAVE_IOCTL;
+        self->readblocks[0] = mp_const_none;
+        self->readblocks[1] = bdev;
+        self->readblocks[2] = (mp_obj_t)sdcardio_sdcard_readblocks; // native version
+        self->writeblocks[0] = mp_const_none;
+        self->writeblocks[1] = bdev;
+        self->writeblocks[2] = (mp_obj_t)sdcardio_sdcard_writeblocks; // native version
+        self->u.ioctl[0] = mp_const_none;
+        self->u.ioctl[1] = bdev;
+        self->u.ioctl[2] = (mp_obj_t)sdcardio_sdcard_ioctl; // native version
+    }
+    #endif
+    #if CIRCUITPY_SDIOIO
+    if (mp_obj_get_type(bdev) == &sdioio_SDCard_type) {
+        // TODO: Enable native blockdev for SDIO too.
+    }
+    #endif
     if (self->u.ioctl[0] != MP_OBJ_NULL) {
         // Device supports new block protocol, so indicate it
         self->flags |= MP_BLOCKDEV_FLAG_HAVE_IOCTL;
@@ -48,8 +77,8 @@ void mp_vfs_blockdev_init(mp_vfs_blockdev_t *self, mp_obj_t bdev) {
 
 int mp_vfs_blockdev_read(mp_vfs_blockdev_t *self, size_t block_num, size_t num_blocks, uint8_t *buf) {
     if (self->flags & MP_BLOCKDEV_FLAG_NATIVE) {
-        mp_uint_t (*f)(uint8_t *, uint32_t, uint32_t) = (void *)(uintptr_t)self->readblocks[2];
-        return f(buf, block_num, num_blocks);
+        mp_uint_t (*f)(mp_obj_t self, uint8_t *, uint32_t, uint32_t) = (void *)(uintptr_t)self->readblocks[2];
+        return f(self->readblocks[1], buf, block_num, num_blocks);
     } else {
         mp_obj_array_t ar = {{&mp_type_bytearray}, BYTEARRAY_TYPECODE, 0, num_blocks *self->block_size, buf};
         self->readblocks[2] = MP_OBJ_NEW_SMALL_INT(block_num);
@@ -80,8 +109,8 @@ int mp_vfs_blockdev_write(mp_vfs_blockdev_t *self, size_t block_num, size_t num_
     }
 
     if (self->flags & MP_BLOCKDEV_FLAG_NATIVE) {
-        mp_uint_t (*f)(const uint8_t *, uint32_t, uint32_t) = (void *)(uintptr_t)self->writeblocks[2];
-        return f(buf, block_num, num_blocks);
+        mp_uint_t (*f)(mp_obj_t self, const uint8_t *, uint32_t, uint32_t) = (void *)(uintptr_t)self->writeblocks[2];
+        return f(self->writeblocks[1], buf, block_num, num_blocks);
     } else {
         mp_obj_array_t ar = {{&mp_type_bytearray}, BYTEARRAY_TYPECODE, 0, num_blocks *self->block_size, (void *)buf};
         self->writeblocks[2] = MP_OBJ_NEW_SMALL_INT(block_num);
@@ -112,6 +141,15 @@ int mp_vfs_blockdev_write_ext(mp_vfs_blockdev_t *self, size_t block_num, size_t
 
 mp_obj_t mp_vfs_blockdev_ioctl(mp_vfs_blockdev_t *self, uintptr_t cmd, uintptr_t arg) {
     if (self->flags & MP_BLOCKDEV_FLAG_HAVE_IOCTL) {
+        if (self->flags & MP_BLOCKDEV_FLAG_NATIVE) {
+            size_t out_value;
+            bool (*f)(mp_obj_t self, uint32_t, uint32_t, size_t *) = (void *)(uintptr_t)self->u.ioctl[2];
+            bool b = f(self->u.ioctl[1], cmd, arg, &out_value);
+            if (!b) {
+                return mp_const_none;
+            }
+            return MP_OBJ_NEW_SMALL_INT(out_value);
+        }
         // New protocol with ioctl
         self->u.ioctl[2] = MP_OBJ_NEW_SMALL_INT(cmd);
         self->u.ioctl[3] = MP_OBJ_NEW_SMALL_INT(arg);

extmod/vfs_fat.h

@@ -34,6 +34,7 @@ typedef struct _fs_user_mount_t {
     mp_obj_base_t base;
     mp_vfs_blockdev_t blockdev;
     FATFS fatfs;
+    int8_t lock_count;
 } fs_user_mount_t;
 
 extern const byte fresult_to_errno_table[20];

locale/circuitpython.pot

@@ -1388,6 +1388,10 @@ msgstr ""
 msgid "Missing jmp_pin. %q[%u] jumps on pin"
 msgstr ""
 
+#: shared-module/storage/__init__.c
+msgid "Mount point missing. Create first (maybe via USB)"
+msgstr ""
+
 #: shared-bindings/busio/UART.c shared-bindings/displayio/Group.c
 msgid "Must be a %q subclass."
 msgstr ""

ports/espressif/supervisor/internal_flash.c

@@ -60,7 +60,8 @@ STATIC uint32_t _cache_lba = 0xffffffff;
 #define SECSIZE(fs) ((fs)->ssize)
 #endif // FF_MAX_SS == FF_MIN_SS
 STATIC DWORD fatfs_bytes(void) {
-    FATFS *fatfs = filesystem_circuitpy();
+    fs_user_mount_t *fs_mount = filesystem_circuitpy();
+    FATFS *fatfs = &fs_mount->fatfs;
     return (fatfs->csize * SECSIZE(fatfs)) * (fatfs->n_fatent - 2);
 }
 STATIC bool storage_extended = true;

shared-bindings/sdcardio/SDCard.c

@@ -131,7 +131,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(sdcardio_sdcard_deinit_obj, sdcardio_sdcard_deinit);
 //|
 //|         :return: None"""
 
-STATIC mp_obj_t sdcardio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
+STATIC mp_obj_t _sdcardio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
     uint32_t start_block = mp_obj_get_int(start_block_in);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(buf_in, &bufinfo, MP_BUFFER_WRITE);
@@ -143,7 +143,7 @@ STATIC mp_obj_t sdcardio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_bloc
     return mp_const_none;
 }
 
-MP_DEFINE_CONST_FUN_OBJ_3(sdcardio_sdcard_readblocks_obj, sdcardio_sdcard_readblocks);
+MP_DEFINE_CONST_FUN_OBJ_3(sdcardio_sdcard_readblocks_obj, _sdcardio_sdcard_readblocks);
 
 //|     def sync(self) -> None:
 //|         """Ensure all blocks written are actually committed to the SD card
@@ -171,7 +171,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(sdcardio_sdcard_sync_obj, sdcardio_sdcard_sync);
 //|         :return: None"""
 //|
 
-STATIC mp_obj_t sdcardio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
+STATIC mp_obj_t _sdcardio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
     uint32_t start_block = mp_obj_get_int(start_block_in);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(buf_in, &bufinfo, MP_BUFFER_READ);
@@ -182,7 +182,7 @@ STATIC mp_obj_t sdcardio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_blo
     }
     return mp_const_none;
 }
-MP_DEFINE_CONST_FUN_OBJ_3(sdcardio_sdcard_writeblocks_obj, sdcardio_sdcard_writeblocks);
+MP_DEFINE_CONST_FUN_OBJ_3(sdcardio_sdcard_writeblocks_obj, _sdcardio_sdcard_writeblocks);
 
 STATIC const mp_rom_map_elem_t sdcardio_sdcard_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_count), MP_ROM_PTR(&sdcardio_sdcard_count_obj) },

shared-bindings/sdcardio/SDCard.h

@@ -27,4 +27,19 @@
 
 #pragma once
 
+#include "shared-module/sdcardio/SDCard.h"
+
 extern const mp_obj_type_t sdcardio_SDCard_type;
+
+void common_hal_sdcardio_sdcard_construct(sdcardio_sdcard_obj_t *self, busio_spi_obj_t *spi, const mcu_pin_obj_t *cs, int baudrate);
+void common_hal_sdcardio_sdcard_deinit(sdcardio_sdcard_obj_t *self);
+void common_hal_sdcardio_sdcard_check_for_deinit(sdcardio_sdcard_obj_t *self);
+int common_hal_sdcardio_sdcard_get_blockcount(sdcardio_sdcard_obj_t *self);
+int common_hal_sdcardio_sdcard_readblocks(sdcardio_sdcard_obj_t *self, uint32_t start_block, mp_buffer_info_t *buf);
+int common_hal_sdcardio_sdcard_sync(sdcardio_sdcard_obj_t *self);
+int common_hal_sdcardio_sdcard_writeblocks(sdcardio_sdcard_obj_t *self, uint32_t start_block, mp_buffer_info_t *buf);
+
+// Used by native vfs blockdev.
+mp_uint_t sdcardio_sdcard_readblocks(mp_obj_t self_in, uint8_t *buf, uint32_t start_block, uint32_t buflen);
+mp_uint_t sdcardio_sdcard_writeblocks(mp_obj_t self_in, uint8_t *buf, uint32_t start_block, uint32_t buflen);
+bool sdcardio_sdcard_ioctl(mp_obj_t self_in, size_t cmd, size_t arg, mp_int_t *out_value);

shared-bindings/sdioio/SDCard.c

@@ -164,7 +164,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(sdioio_sdcard_count_obj, sdioio_sdcard_count);
 //|         :param ~circuitpython_typing.WriteableBuffer buf: The buffer to write into.  Length must be multiple of 512.
 //|
 //|         :return: None"""
-STATIC mp_obj_t sdioio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
+STATIC mp_obj_t _sdioio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
     uint32_t start_block = mp_obj_get_int(start_block_in);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(buf_in, &bufinfo, MP_BUFFER_WRITE);
@@ -176,7 +176,7 @@ STATIC mp_obj_t sdioio_sdcard_readblocks(mp_obj_t self_in, mp_obj_t start_block_
     return mp_const_none;
 }
 
-MP_DEFINE_CONST_FUN_OBJ_3(sdioio_sdcard_readblocks_obj, sdioio_sdcard_readblocks);
+MP_DEFINE_CONST_FUN_OBJ_3(sdioio_sdcard_readblocks_obj, _sdioio_sdcard_readblocks);
 
 //|     def writeblocks(self, start_block: int, buf: ReadableBuffer) -> None:
 //|         """Write one or more blocks to the card
@@ -185,7 +185,7 @@ MP_DEFINE_CONST_FUN_OBJ_3(sdioio_sdcard_readblocks_obj, sdioio_sdcard_readblocks
 //|         :param ~circuitpython_typing.ReadableBuffer buf: The buffer to read from.  Length must be multiple of 512.
 //|
 //|         :return: None"""
-STATIC mp_obj_t sdioio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
+STATIC mp_obj_t _sdioio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_block_in, mp_obj_t buf_in) {
     uint32_t start_block = mp_obj_get_int(start_block_in);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(buf_in, &bufinfo, MP_BUFFER_READ);
@@ -197,7 +197,7 @@ STATIC mp_obj_t sdioio_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t start_block
     return mp_const_none;
 }
 
-MP_DEFINE_CONST_FUN_OBJ_3(sdioio_sdcard_writeblocks_obj, sdioio_sdcard_writeblocks);
+MP_DEFINE_CONST_FUN_OBJ_3(sdioio_sdcard_writeblocks_obj, _sdioio_sdcard_writeblocks);
 
 //|     frequency: int
 //|     """The actual SDIO bus frequency. This may not match the frequency

shared-module/os/getenv.c

@@ -62,8 +62,12 @@ STATIC bool open_file(const char *name, file_arg *active_file) {
         return false;
     }
     #else
-    FATFS *fs = filesystem_circuitpy();
-    FRESULT result = f_open(fs, active_file, name, FA_READ);
+    fs_user_mount_t *fs_mount = filesystem_circuitpy();
+    if (fs_mount == NULL) {
+        return false;
+    }
+    FATFS *fatfs = &fs_mount->fatfs;
+    FRESULT result = f_open(fatfs, active_file, name, FA_READ);
     return result == FR_OK;
     #endif
 }