From f98a20023a3ff00b1a40e17ef4f699e6cfd4cdf5 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 26 Jul 2023 19:19:42 +0200 Subject: [PATCH 01/29] contrib: change tabs to spaces Reverting spacing changes of 1de0476b2e88916356d613b9c90462f6e5ee5e1f --- contrib/genpkglist.py | 428 +++++++++++++++++++++--------------------- 1 file changed, 214 insertions(+), 214 deletions(-) diff --git a/contrib/genpkglist.py b/contrib/genpkglist.py index c50f74e5..6798adef 100644 --- a/contrib/genpkglist.py +++ b/contrib/genpkglist.py @@ -10,87 +10,87 @@ class PackageList: - def __init__(self, name: str, pkgs: list): - self.name = name - self.pkgs = pkgs + def __init__(self, name: str, pkgs: list): + self.name = name + self.pkgs = pkgs - def __repr__(self): - return self.name + def __repr__(self): + return self.name - def __lt__(self, other): - return self.name < other.name + def __lt__(self, other): + return self.name < other.name - def render(self): - return Template(""" + def render(self): + return Template(""" INCLUDE_{{ name }} := \\ {%- for pkg in pkgs %} - {{ pkg }}{% if not loop.last %} \{% endif %} + {{ pkg }}{% if not loop.last %} \{% endif %} {%- endfor %} EXCLUDE_{{ name }} := \\ {%- for pkg in pkgs %} - -{{ pkg }}{% if not loop.last %} \{% endif %} + -{{ pkg }}{% if not loop.last %} \{% endif %} {%- endfor %}""").render( - name=self.name, - pkgs=self.pkgs - ) + name=self.name, + pkgs=self.pkgs + ) class Target: - def __init__(self, name): - self.name = name - self.devices = set() - self.pkglists = set() - self.excludes = defaultdict(set) - - def add_device(self, device: str): - self.devices.add(device) - - def add_pkglist(self, pkglist: PackageList): - self.pkglists.add(pkglist) - return self - - def exclude(self, devices: [str], pkglists: [PackageList]=None): - for device in devices: - assert(device in self.devices), "Device %s not in target %s" % (device, self.name) - if not pkglists: - self.excludes[device] = self.pkglists - else: - self.excludes[device] = self.excludes[device].union(pkglists) - - return self - - def render(self): - if not self.pkglists: - return """ + def __init__(self, name): + self.name = name + self.devices = set() + self.pkglists = set() + self.excludes = defaultdict(set) + + def add_device(self, device: str): + self.devices.add(device) + + def add_pkglist(self, pkglist: PackageList): + self.pkglists.add(pkglist) + return self + + def exclude(self, devices: [str], pkglists: [PackageList]=None): + for device in devices: + assert(device in self.devices), "Device %s not in target %s" % (device, self.name) + if not pkglists: + self.excludes[device] = self.pkglists + else: + self.excludes[device] = self.excludes[device].union(pkglists) + + return self + + def render(self): + if not self.pkglists: + return """ # no pkglists for target %s """ % self.name - return Template(""" + return Template(""" ifeq ($(GLUON_TARGET),{{ target }}) - GLUON_SITE_PACKAGES += {% for pkglist in pkglists %}$(INCLUDE_{{ pkglist.name }}){% if not loop.last %} {% endif %}{% endfor %} + GLUON_SITE_PACKAGES += {% for pkglist in pkglists %}$(INCLUDE_{{ pkglist.name }}){% if not loop.last %} {% endif %}{% endfor %} {% for device, exclude in excludes.items() %} - GLUON_{{ device }}_SITE_PACKAGES += {% for pkglist in exclude|sort %}$(EXCLUDE_{{ pkglist.name }}){% if not loop.last %} {% endif %}{% endfor %} + GLUON_{{ device }}_SITE_PACKAGES += {% for pkglist in exclude|sort %}$(EXCLUDE_{{ pkglist.name }}){% if not loop.last %} {% endif %}{% endfor %} {%- endfor %} endif""").render( - target=self.name, - pkglists=sorted(self.pkglists), - excludes=self.excludes - ) + target=self.name, + pkglists=sorted(self.pkglists), + excludes=self.excludes + ) targets = {} targetdir = os.path.join(GLUON_DIR, 'targets') for targetfile in os.listdir(targetdir): - if targetfile in ['generic', 'targets.mk'] or targetfile.endswith('.inc'): - continue + if targetfile in ['generic', 'targets.mk'] or targetfile.endswith('.inc'): + continue - target = Target(targetfile) - with open(os.path.join(targetdir, targetfile)) as handle: - for line in handle.readlines(): - if line.startswith('device'): - target.add_device(line.split('\'')[1]) + target = Target(targetfile) + with open(os.path.join(targetdir, targetfile)) as handle: + for line in handle.readlines(): + if line.startswith('device'): + target.add_device(line.split('\'')[1]) - targets[targetfile] = target + targets[targetfile] = target # @@ -103,56 +103,56 @@ def render(self): pkglists.append(PKGS_USB) PKGS_USB_HID = PackageList('USB_HID', [ - 'kmod-usb-hid', - 'kmod-hid-generic' + 'kmod-usb-hid', + 'kmod-hid-generic' ]) pkglists.append(PKGS_USB_HID) PKGS_USB_SERIAL = PackageList('USB_SERIAL', [ - 'kmod-usb-serial', - 'kmod-usb-serial-ftdi', - 'kmod-usb-serial-pl2303' + 'kmod-usb-serial', + 'kmod-usb-serial-ftdi', + 'kmod-usb-serial-pl2303' ]) pkglists.append(PKGS_USB_SERIAL) PKGS_USB_STORAGE = PackageList('USB_STORAGE', [ - 'block-mount', - 'blkid', - 'kmod-fs-ext4', - 'kmod-fs-ntfs', - 'kmod-fs-vfat', - 'kmod-usb-storage', - 'kmod-usb-storage-extras', # Card Readers - 'kmod-usb-storage-uas', # USB Attached SCSI (UAS/UASP) - 'kmod-nls-base', - 'kmod-nls-cp1250', # NLS Codepage 1250 (Eastern Europe) - 'kmod-nls-cp437', # NLS Codepage 437 (United States, Canada) - 'kmod-nls-cp850', # NLS Codepage 850 (Europe) - 'kmod-nls-cp852', # NLS Codepage 852 (Europe) - 'kmod-nls-iso8859-1', # NLS ISO 8859-1 (Latin 1) - 'kmod-nls-iso8859-13', # NLS ISO 8859-13 (Latin 7; Baltic) - 'kmod-nls-iso8859-15', # NLS ISO 8859-15 (Latin 9) - 'kmod-nls-iso8859-2', # NLS ISO 8859-2 (Latin 2) - 'kmod-nls-utf8' # NLS UTF-8 + 'block-mount', + 'blkid', + 'kmod-fs-ext4', + 'kmod-fs-ntfs', + 'kmod-fs-vfat', + 'kmod-usb-storage', + 'kmod-usb-storage-extras', # Card Readers + 'kmod-usb-storage-uas', # USB Attached SCSI (UAS/UASP) + 'kmod-nls-base', + 'kmod-nls-cp1250', # NLS Codepage 1250 (Eastern Europe) + 'kmod-nls-cp437', # NLS Codepage 437 (United States, Canada) + 'kmod-nls-cp850', # NLS Codepage 850 (Europe) + 'kmod-nls-cp852', # NLS Codepage 852 (Europe) + 'kmod-nls-iso8859-1', # NLS ISO 8859-1 (Latin 1) + 'kmod-nls-iso8859-13', # NLS ISO 8859-13 (Latin 7; Baltic) + 'kmod-nls-iso8859-15', # NLS ISO 8859-15 (Latin 9) + 'kmod-nls-iso8859-2', # NLS ISO 8859-2 (Latin 2) + 'kmod-nls-utf8' # NLS UTF-8 ]) pkglists.append(PKGS_USB_STORAGE) PKGS_USB_NET = PackageList('USB_NET', [ - 'kmod-mii', - 'kmod-usb-net', - 'kmod-usb-net-asix', - 'kmod-usb-net-asix-ax88179', - 'kmod-usb-net-cdc-eem', - 'kmod-usb-net-cdc-ether', - 'kmod-usb-net-cdc-subset', - 'kmod-usb-net-dm9601-ether', - 'kmod-usb-net-hso', - 'kmod-usb-net-ipheth', - 'kmod-usb-net-mcs7830', - 'kmod-usb-net-pegasus', - 'kmod-usb-net-rndis', - 'kmod-usb-net-rtl8152', - 'kmod-usb-net-smsc95xx' + 'kmod-mii', + 'kmod-usb-net', + 'kmod-usb-net-asix', + 'kmod-usb-net-asix-ax88179', + 'kmod-usb-net-cdc-eem', + 'kmod-usb-net-cdc-ether', + 'kmod-usb-net-cdc-subset', + 'kmod-usb-net-dm9601-ether', + 'kmod-usb-net-hso', + 'kmod-usb-net-ipheth', + 'kmod-usb-net-mcs7830', + 'kmod-usb-net-pegasus', + 'kmod-usb-net-rndis', + 'kmod-usb-net-rtl8152', + 'kmod-usb-net-smsc95xx' ]) pkglists.append(PKGS_USB_NET) @@ -160,13 +160,13 @@ def render(self): pkglists.append(PKGS_PCI) PKGS_PCI_NET = PackageList('PCI_NET', [ - 'kmod-bnx2' # Broadcom NetExtreme BCM5706/5708/5709/5716 + 'kmod-bnx2' # Broadcom NetExtreme BCM5706/5708/5709/5716 ]) pkglists.append(PKGS_PCI_NET) PKGS_TLS = PackageList('TLS', [ - 'ca-bundle', - 'libustream-openssl' + 'ca-bundle', + 'libustream-openssl' ]) pkglists.append(PKGS_TLS) @@ -180,88 +180,88 @@ def render(self): # targets.get('ar71xx-generic'). \ - add_pkglist(PKGS_USB). \ - add_pkglist(PKGS_USB_NET). \ - add_pkglist(PKGS_USB_SERIAL). \ - add_pkglist(PKGS_USB_STORAGE). \ - add_pkglist(PKGS_TLS). \ - exclude([ # devices without usb ports - 'allnet-all0315n', - 'avm-fritz-wlan-repeater-300e', - 'avm-fritz-wlan-repeater-450e', - 'd-link-dap-1330-rev-a1', - 'd-link-dir-825-rev-b1', - 'meraki-mr12', - 'meraki-mr16', - 'ocedo-koala', - 'openmesh-mr1750', - 'openmesh-mr600', - 'openmesh-mr900', - 'openmesh-om2p', - 'openmesh-om5p', - 'openmesh-om5p-ac', - 'tp-link-cpe210-v1', - 'tp-link-cpe210-v2', - 'tp-link-cpe210-v3', - 'tp-link-cpe510-v1', - 'tp-link-wbs210-v1', - 'tp-link-wbs510-v1', - 'tp-link-archer-c25-v1', - 'tp-link-archer-c58-v1', - 'tp-link-archer-c60-v1', - 'tp-link-archer-c60-v2', - 'tp-link-re355', - 'tp-link-re450', - 'ubiquiti-airgateway', - 'ubiquiti-airgateway-pro', - 'ubiquiti-bullet-m', - 'ubiquiti-rocket-m', - 'ubiquiti-nanostation-m', - 'ubiquiti-loco-m-xw', - 'ubiquiti-nanostation-m-xw', - 'ubiquiti-rocket-m-xw', - 'ubiquiti-rocket-m-ti', - 'ubiquiti-unifi', - 'ubiquiti-unifi-ap-pro', - 'ubiquiti-unifiap-outdoor', - 'ubiquiti-unifiap-outdoor+', - 'ubiquiti-ls-sr71', - 'ubiquiti-unifi-ac-lite', - 'ubiquiti-unifi-ac-lr', - 'ubiquiti-unifi-ac-pro', - 'ubiquiti-unifi-ac-mesh'], pkglists=[PKGS_USB, PKGS_USB_NET, PKGS_USB_SERIAL, PKGS_USB_STORAGE]). \ - exclude([ # devices with less than 64M memory - 'linksys-wrt160nl', - 'tp-link-tl-wr710n-v1', - 'tp-link-tl-wr710n-v2.1', - 'tp-link-tl-wr842n-nd-v1', - 'tp-link-tl-wr842n-nd-v2', - 'tp-link-tl-wr1043n-nd-v1', - 'ubiquiti-airrouter']) + add_pkglist(PKGS_USB). \ + add_pkglist(PKGS_USB_NET). \ + add_pkglist(PKGS_USB_SERIAL). \ + add_pkglist(PKGS_USB_STORAGE). \ + add_pkglist(PKGS_TLS). \ + exclude([ # devices without usb ports + 'allnet-all0315n', + 'avm-fritz-wlan-repeater-300e', + 'avm-fritz-wlan-repeater-450e', + 'd-link-dap-1330-rev-a1', + 'd-link-dir-825-rev-b1', + 'meraki-mr12', + 'meraki-mr16', + 'ocedo-koala', + 'openmesh-mr1750', + 'openmesh-mr600', + 'openmesh-mr900', + 'openmesh-om2p', + 'openmesh-om5p', + 'openmesh-om5p-ac', + 'tp-link-cpe210-v1', + 'tp-link-cpe210-v2', + 'tp-link-cpe210-v3', + 'tp-link-cpe510-v1', + 'tp-link-wbs210-v1', + 'tp-link-wbs510-v1', + 'tp-link-archer-c25-v1', + 'tp-link-archer-c58-v1', + 'tp-link-archer-c60-v1', + 'tp-link-archer-c60-v2', + 'tp-link-re355', + 'tp-link-re450', + 'ubiquiti-airgateway', + 'ubiquiti-airgateway-pro', + 'ubiquiti-bullet-m', + 'ubiquiti-rocket-m', + 'ubiquiti-nanostation-m', + 'ubiquiti-loco-m-xw', + 'ubiquiti-nanostation-m-xw', + 'ubiquiti-rocket-m-xw', + 'ubiquiti-rocket-m-ti', + 'ubiquiti-unifi', + 'ubiquiti-unifi-ap-pro', + 'ubiquiti-unifiap-outdoor', + 'ubiquiti-unifiap-outdoor+', + 'ubiquiti-ls-sr71', + 'ubiquiti-unifi-ac-lite', + 'ubiquiti-unifi-ac-lr', + 'ubiquiti-unifi-ac-pro', + 'ubiquiti-unifi-ac-mesh'], pkglists=[PKGS_USB, PKGS_USB_NET, PKGS_USB_SERIAL, PKGS_USB_STORAGE]). \ + exclude([ # devices with less than 64M memory + 'linksys-wrt160nl', + 'tp-link-tl-wr710n-v1', + 'tp-link-tl-wr710n-v2.1', + 'tp-link-tl-wr842n-nd-v1', + 'tp-link-tl-wr842n-nd-v2', + 'tp-link-tl-wr1043n-nd-v1', + 'ubiquiti-airrouter']) targets.get('ath79-generic'). \ - add_pkglist(PKGS_USB). \ - add_pkglist(PKGS_USB_NET). \ - add_pkglist(PKGS_USB_SERIAL). \ - add_pkglist(PKGS_USB_STORAGE). \ - add_pkglist(PKGS_TLS). \ - exclude([ # devices without usb ports - 'devolo-wifi-pro-1200e', - 'devolo-wifi-pro-1200i', - 'devolo-wifi-pro-1750c', - 'devolo-wifi-pro-1750i', - 'devolo-wifi-pro-1750x', - 'ocedo-raccoon', - 'tp-link-archer-c6-v2', - 'tp-link-cpe220-v3'], pkglists=[PKGS_USB, PKGS_USB_NET, PKGS_USB_SERIAL, PKGS_USB_STORAGE]) + add_pkglist(PKGS_USB). \ + add_pkglist(PKGS_USB_NET). \ + add_pkglist(PKGS_USB_SERIAL). \ + add_pkglist(PKGS_USB_STORAGE). \ + add_pkglist(PKGS_TLS). \ + exclude([ # devices without usb ports + 'devolo-wifi-pro-1200e', + 'devolo-wifi-pro-1200i', + 'devolo-wifi-pro-1750c', + 'devolo-wifi-pro-1750i', + 'devolo-wifi-pro-1750x', + 'ocedo-raccoon', + 'tp-link-archer-c6-v2', + 'tp-link-cpe220-v3'], pkglists=[PKGS_USB, PKGS_USB_NET, PKGS_USB_SERIAL, PKGS_USB_STORAGE]) for target in ['ar71xx-nand', 'ipq40xx-generic', 'ipq806x-generic', 'lantiq-xway', 'lantiq-xrx200', 'mpc85xx-generic', 'mpc85xx-p1020', 'mvebu-cortexa9', 'ramips-mt7620', 'sunxi-cortexa7']: - targets.get(target). \ - add_pkglist(PKGS_USB). \ - add_pkglist(PKGS_USB_NET). \ - add_pkglist(PKGS_USB_SERIAL). \ - add_pkglist(PKGS_USB_STORAGE). \ - add_pkglist(PKGS_TLS) + targets.get(target). \ + add_pkglist(PKGS_USB). \ + add_pkglist(PKGS_USB_NET). \ + add_pkglist(PKGS_USB_SERIAL). \ + add_pkglist(PKGS_USB_STORAGE). \ + add_pkglist(PKGS_TLS) targets.get('mpc85xx-p1020').add_pkglist(PKGS_TLS) @@ -270,53 +270,53 @@ def render(self): add_pkglist(PKGS_EOL) for target in ['brcm2708-bcm2708', 'brcm2708-bcm2709', 'brcm2708-bcm2710']: - targets.get(target). \ - add_pkglist(PKGS_USB). \ - add_pkglist(PKGS_USB_NET). \ - add_pkglist(PKGS_USB_SERIAL). \ - add_pkglist(PKGS_USB_STORAGE). \ - add_pkglist(PKGS_USB_HID). \ - add_pkglist(PKGS_TLS) + targets.get(target). \ + add_pkglist(PKGS_USB). \ + add_pkglist(PKGS_USB_NET). \ + add_pkglist(PKGS_USB_SERIAL). \ + add_pkglist(PKGS_USB_STORAGE). \ + add_pkglist(PKGS_USB_HID). \ + add_pkglist(PKGS_TLS) targets.get('ramips-mt7621'). \ - add_pkglist(PKGS_USB). \ - add_pkglist(PKGS_USB_NET). \ - add_pkglist(PKGS_USB_SERIAL). \ - add_pkglist(PKGS_USB_STORAGE). \ - add_pkglist(PKGS_TLS). \ - exclude([ # devices without usb ports - 'netgear-ex6150', - 'ubnt-erx', - 'ubnt-erx-sfp'], pkglists=[PKGS_USB, PKGS_USB_NET, PKGS_USB_SERIAL, PKGS_USB_STORAGE]) + add_pkglist(PKGS_USB). \ + add_pkglist(PKGS_USB_NET). \ + add_pkglist(PKGS_USB_SERIAL). \ + add_pkglist(PKGS_USB_STORAGE). \ + add_pkglist(PKGS_TLS). \ + exclude([ # devices without usb ports + 'netgear-ex6150', + 'ubnt-erx', + 'ubnt-erx-sfp'], pkglists=[PKGS_USB, PKGS_USB_NET, PKGS_USB_SERIAL, PKGS_USB_STORAGE]) targets.get('ramips-mt76x8'). \ - add_pkglist(PKGS_USB). \ - add_pkglist(PKGS_USB_NET). \ - add_pkglist(PKGS_USB_SERIAL). \ - add_pkglist(PKGS_USB_STORAGE). \ - add_pkglist(PKGS_TLS). \ - exclude([ # devices without usb ports - 'cudy-wr1000', - 'gl.inet-vixmini', - 'tp-link-archer-c50-v3', - 'tp-link-archer-c50-v4', - 'tp-link-tl-wa801nd-v5', - 'tp-link-tl-wr841n-v13'], pkglists=[PKGS_USB, PKGS_USB_NET, PKGS_USB_SERIAL, PKGS_USB_STORAGE]) + add_pkglist(PKGS_USB). \ + add_pkglist(PKGS_USB_NET). \ + add_pkglist(PKGS_USB_SERIAL). \ + add_pkglist(PKGS_USB_STORAGE). \ + add_pkglist(PKGS_TLS). \ + exclude([ # devices without usb ports + 'cudy-wr1000', + 'gl.inet-vixmini', + 'tp-link-archer-c50-v3', + 'tp-link-archer-c50-v4', + 'tp-link-tl-wa801nd-v5', + 'tp-link-tl-wr841n-v13'], pkglists=[PKGS_USB, PKGS_USB_NET, PKGS_USB_SERIAL, PKGS_USB_STORAGE]) for target in ['x86-64', 'x86-generic', 'x86-geode']: - targets.get(target). \ - add_pkglist(PKGS_USB). \ - add_pkglist(PKGS_USB_NET). \ - add_pkglist(PKGS_USB_SERIAL). \ - add_pkglist(PKGS_USB_STORAGE). \ - add_pkglist(PKGS_PCI). \ - add_pkglist(PKGS_PCI_NET). \ - add_pkglist(PKGS_TLS) + targets.get(target). \ + add_pkglist(PKGS_USB). \ + add_pkglist(PKGS_USB_NET). \ + add_pkglist(PKGS_USB_SERIAL). \ + add_pkglist(PKGS_USB_STORAGE). \ + add_pkglist(PKGS_PCI). \ + add_pkglist(PKGS_PCI_NET). \ + add_pkglist(PKGS_TLS) if __name__ == '__main__': - for pkglist in pkglists: - print(pkglist.render()) + for pkglist in pkglists: + print(pkglist.render()) - for target in sorted(targets.values(), key=lambda x: x.name): - print(target.render()) \ No newline at end of file + for target in sorted(targets.values(), key=lambda x: x.name): + print(target.render()) From 6c18012eea90f57f9d320d11bbaf3e87d82eea3e Mon Sep 17 00:00:00 2001 From: Malte Wedel Date: Wed, 30 Mar 2022 21:26:47 +0000 Subject: [PATCH 02/29] Backport changes from v2022.2.1-next3 - Remove mesh_on_wan from site.conf - default is false anyway --- site.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/site.conf b/site.conf index 2ace0eb5..7d3652d7 100644 --- a/site.conf +++ b/site.conf @@ -13,8 +13,6 @@ timezone = 'CET-1CEST,M3.5.0,M10.5.0/3', ntp_servers = { 'ntp.ext.ffmuc.net' }, - mesh_on_wan = false, - regdom = 'DE', wifi24 = { From 4d499a10b923eecfdb016954220a8ecc90af43c7 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 14 Sep 2022 11:49:11 +0200 Subject: [PATCH 03/29] Move install_build_dependencies.sh to scripts/ --- .github/workflows/firmware.yml | 2 +- .../install_build_dependencies.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename install_build_dependencies.sh => scripts/install_build_dependencies.sh (100%) diff --git a/.github/workflows/firmware.yml b/.github/workflows/firmware.yml index c7a3534c..ac0e69a0 100644 --- a/.github/workflows/firmware.yml +++ b/.github/workflows/firmware.yml @@ -33,7 +33,7 @@ jobs: fetch-depth: 0 - name: Install build dependencies run: | - sudo -E ./install_build_dependencies.sh + sudo -E ./scripts/install_build_dependencies.sh - name: build target ${{ matrix.target }} id: compile run: | diff --git a/install_build_dependencies.sh b/scripts/install_build_dependencies.sh similarity index 100% rename from install_build_dependencies.sh rename to scripts/install_build_dependencies.sh From 2ce76bd0ef914b113bb752ab512b577dba962215 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 14 Sep 2022 11:50:14 +0200 Subject: [PATCH 04/29] Add dependencies to README --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 5a1761bd..e99d2caf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ ![Build ffmuc firmware](https://github.com/freifunkMUC/site-ffm/workflows/Build%20ffmuc%20firmware/badge.svg) +## Dependencies + +* git +* GNU make +* GNU patch +* [Upstream Gluon dependencies](https://gluon.readthedocs.io/en/latest/user/getting_started.html#dependencies) + +For convenience, you can find a script to install those dependencies on an Ubuntu-based distribution:\ + [install_build_dependencies.sh](scripts/install_build_dependencies.sh) + ## Building Check out this repository and execute `make`, i.e. like this: From 994a8d0510ff510cb7cc43894a7afabc2a80610d Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 14 Sep 2022 11:26:51 +0200 Subject: [PATCH 05/29] Fix hardcoded gluon-build path --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2038a814..9b265c70 100644 --- a/Makefile +++ b/Makefile @@ -96,7 +96,7 @@ gluon-patch: (git branch -D patched) \ fi (cd ${GLUON_BUILD_DIR}; git checkout -B patching) - if [ -d "gluon-build/site/patches" -a "gluon-build/site/patches/*.patch" ]; then \ + if [ -d "${GLUON_BUILD_DIR}/site/patches" -a "${GLUON_BUILD_DIR}/site/patches/*.patch" ]; then \ (cd ${GLUON_BUILD_DIR}; git apply --ignore-space-change --ignore-whitespace --whitespace=nowarn --verbose site/patches/*.patch) || ( \ cd ${GLUON_BUILD_DIR}; \ git clean -fd; \ From c1cea0f006cb5653383da0f254aa5038470bc762 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 14 Sep 2022 13:28:50 +0200 Subject: [PATCH 06/29] Simplify patching and move it into separate script * Make the script idempotent (you can run it again to test patching) * Drop the usage of git branches for patching. The branches were not used and add unncessary complexity * Do not forcefully reset custom commits in gluon-build as part of the patching * Reset the gluon-build environment before applying patches * Do not reset gluon-build environment for debugging if patching failed --- Makefile | 21 +++-------------- scripts/apply_patches.sh | 50 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 18 deletions(-) create mode 100755 scripts/apply_patches.sh diff --git a/Makefile b/Makefile index 9b265c70..2981b4f3 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ GLUON_BUILD_DIR := gluon-build GLUON_GIT_URL := https://github.com/freifunk-gluon/gluon.git GLUON_GIT_REF := v2021.1.2 -PATCH_DIR := ${GLUON_BUILD_DIR}/site/patches +PATCH_DIR := ./patches SECRET_KEY_FILE ?= ${HOME}/.gluon-secret-key GLUON_TARGETS ?= \ @@ -85,27 +85,12 @@ gluon-prepare: output-clean ${GLUON_BUILD_DIR} && rm -rf packages \ && git checkout -q --force ${GLUON_GIT_REF} \ && git clean -fd; - ln -sfT .. ${GLUON_BUILD_DIR}/site make gluon-patch + ln -sfT .. ${GLUON_BUILD_DIR}/site ${GLUON_MAKE} update gluon-patch: - echo "Applying Patches ..." - (cd ${GLUON_BUILD_DIR}) - if [ `git branch --list patched` ]; then \ - (git branch -D patched) \ - fi - (cd ${GLUON_BUILD_DIR}; git checkout -B patching) - if [ -d "${GLUON_BUILD_DIR}/site/patches" -a "${GLUON_BUILD_DIR}/site/patches/*.patch" ]; then \ - (cd ${GLUON_BUILD_DIR}; git apply --ignore-space-change --ignore-whitespace --whitespace=nowarn --verbose site/patches/*.patch) || ( \ - cd ${GLUON_BUILD_DIR}; \ - git clean -fd; \ - git checkout -B patched; \ - git branch -D patching; \ - exit 1 \ - ) \ - fi - (cd ${GLUON_BUILD_DIR}; git branch -M patched) + scripts/apply_patches.sh ${GLUON_BUILD_DIR} ${PATCH_DIR} gluon-clean: rm -rf ${GLUON_BUILD_DIR} diff --git a/scripts/apply_patches.sh b/scripts/apply_patches.sh new file mode 100755 index 00000000..0c4cab04 --- /dev/null +++ b/scripts/apply_patches.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -eEu +set -o pipefail +shopt -s nullglob + +gluon_build_dir=${1:-gluon-build} +gluon_patch_dir="${2:-patches}" + +function reset_gluon_build_dir() { + # Make sure we are in the correct folder + if [[ ! $(pwd) =~ .*${gluon_build_dir} ]]; then + echo "Resetting environment in the wrong folder. Aborting." + return 1 + fi + echo "Resetting environment." + + # Reset all files known to git, but keep manually commited changes. + git checkout . + # Delete all files not known to git + git clean -dx --force + echo "Environment reset." +} + +# Relative patches folder does not work with git-apply below. Make sure it is an absolute path. +if [[ ! ${gluon_patch_dir} =~ ^/ ]]; then + gluon_patch_dir="${PWD}/${gluon_patch_dir}" + echo "Setting patch directory to ${gluon_patch_dir}" +fi + +pushd "${gluon_build_dir}" + +# Check if there are any patches at all +if ! compgen -G "${gluon_patch_dir}/*.patch" >/dev/null; then + echo "No patches found in ${gluon_patch_dir}/*.patch" + exit 1 +fi + +# Reset previously applied patches +reset_gluon_build_dir + +# Apply all patches +echo "Applying Patches." +if ! git apply --ignore-space-change --ignore-whitespace --whitespace=nowarn --verbose "${gluon_patch_dir}"/*.patch; then + echo "Patching failed. Inspect ${gluon_build_dir} folder for failed patches." + exit 1 +fi + +echo "Patching finished." +popd From 927ab82fa784ee7a42d63bee705ea9cc80e0bdc0 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 14 Sep 2022 15:16:38 +0200 Subject: [PATCH 07/29] Add Github Action for bash linting --- .github/workflows/shellcheck.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/shellcheck.yml diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml new file mode 100644 index 00000000..f1317d99 --- /dev/null +++ b/.github/workflows/shellcheck.yml @@ -0,0 +1,23 @@ +--- +name: Shell Linting +on: + workflow_dispatch: + push: + branches: + - stable + - next + pull_request: + +jobs: + shellcheck: + name: runner / shellcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: shellcheck + # Make sure the action is pinned to a commit, as all reviewdog repos + # have hundreds of contributors with write access (breaks easy/often) + uses: reviewdog/action-shellcheck@66c9a47bf02255b250284a82251cb4cadf5043f5 # v1.15.0 + with: + filter_mode: "file" + fail_on_error: true From 89d1189f5a692db4467550007db7c99eb900d9e0 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 14 Sep 2022 15:34:24 +0200 Subject: [PATCH 08/29] Fix in POSIX sh, set flag -E is undefined --- scripts/install_build_dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_build_dependencies.sh b/scripts/install_build_dependencies.sh index 2d444376..62b6fe27 100755 --- a/scripts/install_build_dependencies.sh +++ b/scripts/install_build_dependencies.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh -set -eEux +set -eux # Verify that the script is running in Ubuntu . /etc/lsb-release From cd1fca7ccd22828dfa3c8cd4aa72499f07e8af2c Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 12 Sep 2022 12:39:54 +0200 Subject: [PATCH 09/29] Fix overriding the main git remote by accident In case the gluon-build folder exists and does not contain a .git directory (or file) the following command "git remote set-url origin" will override the git remote of the "main" folder/directory. Avoiding this by checking explicitely for the .git directory inside gluon-build and restructuring the cloning/fetching process. This also speeds up the initial cloning by only fetching the commit required to build. Important legacy-only change: adding git clean -ffdx packages/ --- Makefile | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 2981b4f3..490afbce 100644 --- a/Makefile +++ b/Makefile @@ -76,15 +76,21 @@ sign: manifest ${GLUON_BUILD_DIR}/contrib/sign.sh ${SECRET_KEY_FILE} output/images/sysupgrade/${GLUON_AUTOUPDATER_BRANCH}.manifest ${GLUON_BUILD_DIR}: - git clone ${GLUON_GIT_URL} ${GLUON_BUILD_DIR} - -gluon-prepare: output-clean ${GLUON_BUILD_DIR} - cd ${GLUON_BUILD_DIR} \ - && git remote set-url origin ${GLUON_GIT_URL} \ - && git fetch origin \ - && rm -rf packages \ - && git checkout -q --force ${GLUON_GIT_REF} \ - && git clean -fd; + mkdir -p ${GLUON_BUILD_DIR} + +# Note: "|" means "order only", e.g. "do not care about folder timestamps" +# https://www.gnu.org/savannah-checkouts/gnu/make/manual/html_node/Prerequisite-Types.html +${GLUON_BUILD_DIR}/.git: | ${GLUON_BUILD_DIR} + git init ${GLUON_BUILD_DIR} + cd ${GLUON_BUILD_DIR} && git remote add origin ${GLUON_GIT_URL} + +gluon-update: | ${GLUON_BUILD_DIR}/.git + cd ${GLUON_BUILD_DIR} && git fetch origin ${GLUON_GIT_REF} + cd ${GLUON_BUILD_DIR} && git reset --hard FETCH_HEAD + cd ${GLUON_BUILD_DIR} && git clean -ffdx packages/ # double -f -f required to remove sub-repositories + cd ${GLUON_BUILD_DIR} && git clean -fd + +gluon-prepare: gluon-update output-clean make gluon-patch ln -sfT .. ${GLUON_BUILD_DIR}/site ${GLUON_MAKE} update From 52c7e47d0e3ce86700cccf8f98232aa86b8919fb Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 14 Sep 2022 14:58:43 +0200 Subject: [PATCH 10/29] Run output-clean only during build, not before. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 490afbce..b2ea8fcf 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ info: @echo '# Building release ${GLUON_RELEASE} for branch ${GLUON_AUTOUPDATER_BRANCH}' @echo -build: gluon-prepare +build: gluon-prepare output-clean for target in ${GLUON_TARGETS}; do \ echo ""Building target $$target""; \ ${GLUON_MAKE} download all GLUON_TARGET="$$target"; \ @@ -90,7 +90,7 @@ gluon-update: | ${GLUON_BUILD_DIR}/.git cd ${GLUON_BUILD_DIR} && git clean -ffdx packages/ # double -f -f required to remove sub-repositories cd ${GLUON_BUILD_DIR} && git clean -fd -gluon-prepare: gluon-update output-clean +gluon-prepare: gluon-update make gluon-patch ln -sfT .. ${GLUON_BUILD_DIR}/site ${GLUON_MAKE} update From d0653aba112df2e85c7b55f072c76d85c1617169 Mon Sep 17 00:00:00 2001 From: Christian Date: Tue, 6 Sep 2022 12:47:09 +0200 Subject: [PATCH 11/29] contrib: Update render templates Based on https://git.darmstadt.ccc.de/ffda/firmware/site/-/commit/b7d6ac615d4b09cf2999d97d7ac3ef2bbc9fe060 --- contrib/genpkglist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/genpkglist.py b/contrib/genpkglist.py index 6798adef..4e144579 100644 --- a/contrib/genpkglist.py +++ b/contrib/genpkglist.py @@ -24,12 +24,12 @@ def render(self): return Template(""" INCLUDE_{{ name }} := \\ {%- for pkg in pkgs %} - {{ pkg }}{% if not loop.last %} \{% endif %} + {{ pkg }}{% if not loop.last %} \\{% endif %} {%- endfor %} EXCLUDE_{{ name }} := \\ {%- for pkg in pkgs %} - -{{ pkg }}{% if not loop.last %} \{% endif %} + -{{ pkg }}{% if not loop.last %} \\{% endif %} {%- endfor %}""").render( name=self.name, pkgs=self.pkgs From e29098834e8dc8db4aa22ce5c1910bf2a2eba509 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 26 Jul 2023 15:18:52 +0200 Subject: [PATCH 12/29] Format firmware.yml --- .github/workflows/firmware.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/firmware.yml b/.github/workflows/firmware.yml index ac0e69a0..e0a2c276 100644 --- a/.github/workflows/firmware.yml +++ b/.github/workflows/firmware.yml @@ -13,12 +13,12 @@ jobs: outputs: target_json: ${{ steps.set_target.outputs.target }} steps: - - name: Set target matrix - id: set_target - shell: bash - run: | - target_list="[\"ar71xx-generic\", \"ar71xx-tiny\", \"ar71xx-nand\", \"ath79-generic\", \"brcm2708-bcm2708\", \"brcm2708-bcm2709\", \"ipq40xx-generic\", \"ipq806x-generic\", \"lantiq-xrx200\", \"lantiq-xway\", \"mpc85xx-generic\", \"mpc85xx-p1020\", \"ramips-mt7620\", \"ramips-mt7621\", \"ramips-mt76x8\", \"ramips-rt305x\", \"sunxi-cortexa7\", \"x86-generic\", \"x86-geode\", \"x86-legacy\", \"x86-64\", \"ar71xx-mikrotik\", \"brcm2708-bcm2710\", \"mvebu-cortexa9\"]" - echo ::set-output name=target::{\"target\": $(echo $target_list)}\" + - name: Set target matrix + id: set_target + shell: bash + run: | + target_list="[\"ar71xx-generic\", \"ar71xx-tiny\", \"ar71xx-nand\", \"ath79-generic\", \"brcm2708-bcm2708\", \"brcm2708-bcm2709\", \"ipq40xx-generic\", \"ipq806x-generic\", \"lantiq-xrx200\", \"lantiq-xway\", \"mpc85xx-generic\", \"mpc85xx-p1020\", \"ramips-mt7620\", \"ramips-mt7621\", \"ramips-mt76x8\", \"ramips-rt305x\", \"sunxi-cortexa7\", \"x86-generic\", \"x86-geode\", \"x86-legacy\", \"x86-64\", \"ar71xx-mikrotik\", \"brcm2708-bcm2710\", \"mvebu-cortexa9\"]" + echo ::set-output name=target::{\"target\": $(echo $target_list)}\" build_firmware: needs: generate_target_matrix @@ -52,7 +52,7 @@ jobs: needs: build_firmware if: github.event_name != 'pull_request' outputs: - output1: ${{ steps.create_release.outputs.upload_url }} + output1: ${{ steps.create_release.outputs.upload_url }} steps: - name: Create Release id: create_release @@ -70,7 +70,9 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.generate_target_matrix.outputs.target_json) }} runs-on: ubuntu-latest - needs: [ create_release, generate_target_matrix ] + needs: + - create_release + - generate_target_matrix if: github.event_name != 'pull_request' steps: - name: Download Artifact ${{ matrix.target }} From a15cf7e7338e45a10b524d34e3b796dac1f3b0a6 Mon Sep 17 00:00:00 2001 From: Christian Date: Sun, 18 Sep 2022 16:53:07 +0200 Subject: [PATCH 13/29] Do not build firmware on unrelated changes --- .github/workflows/firmware.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/firmware.yml b/.github/workflows/firmware.yml index e0a2c276..49dce3aa 100644 --- a/.github/workflows/firmware.yml +++ b/.github/workflows/firmware.yml @@ -6,6 +6,11 @@ on: tags: - v* pull_request: + paths-ignore: + - '**.md' + - '**/CODEOWNERS' + - '.github/ISSUE_TEMPLATE' + - '.github/*.yml' jobs: generate_target_matrix: From 15d5a13ff3b8353cea1359ce20c347915b6d2698 Mon Sep 17 00:00:00 2001 From: Christian Date: Sun, 18 Sep 2022 17:05:44 +0200 Subject: [PATCH 14/29] Run shellcheck workflow only for shellscripts --- .github/workflows/shellcheck.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index f1317d99..8744f86e 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -6,7 +6,11 @@ on: branches: - stable - next + paths: + - '**.sh' pull_request: + paths: + - '**.sh' jobs: shellcheck: From 02e189d05f998f7157f87b8de3c0a5b946e01e48 Mon Sep 17 00:00:00 2001 From: Christian Date: Sat, 21 May 2022 12:03:39 +0200 Subject: [PATCH 15/29] Add building instructions with Docker --- Dockerfile_build | 4 ++++ README.md | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 Dockerfile_build diff --git a/Dockerfile_build b/Dockerfile_build new file mode 100644 index 00000000..023c43cb --- /dev/null +++ b/Dockerfile_build @@ -0,0 +1,4 @@ +FROM ubuntu:20.04 + +ADD ./scripts/install_build_dependencies.sh / +RUN /install_build_dependencies.sh diff --git a/README.md b/README.md index e99d2caf..f40c6ebc 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,26 @@ git checkout -b patched && git checkout stable make ``` +### Containerised building + +As the CI is using Ubuntu, only the Ubuntu dependencies are being tracked. To simplify building on other distros, containerised building is also possible: +```sh +docker build -t site-ffm -f Dockerfile_build . +``` +This will build the build Docker image. With the following export, the Makefile will then use the repo for building but will run inside an Ubuntu container.\ +**Note**: If the working directory is a git worktree, add a volume mount for the main git folder. +```sh +docker run --rm -v $(pwd):/site-ffm:ro -v $(pwd)/gluon-build:/site-ffm/gluon-build:rw -v $(pwd)/output:/site-ffm/output:rw -w /site-ffm -u $UID site-ffm-next make +``` + +#### Example +Full command for a [lantiq-xrx200](https://github.com/freifunk-gluon/gluon/blob/v2022.1/targets/lantiq-xrx200) build: + +```sh +mkdir -p logs +docker run --rm -v $(pwd):/site-ffm:ro -v $(pwd)/gluon-build:/site-ffm/gluon-build:rw -v $(pwd)/output:/site-ffm/output:rw -w /site-ffm -u $UID site-ffm-next make V=s BROKEN=1 GLUON_TARGETS=lantiq-xrx200 |& tee logs/buildtest_lantiq-xrx200_$(date --iso=s).log +``` + ## Further Resources This firmware is based on [Gluon](https://gluon.readthedocs.io/en/v2021.1/). From f143e2c59c5401bbc55eff9ba7ec1e759cc95c52 Mon Sep 17 00:00:00 2001 From: Grische Date: Sun, 9 Oct 2022 10:27:23 +0200 Subject: [PATCH 16/29] Adding signing key for grische --- site.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/site.conf b/site.conf index 7d3652d7..627e1cfe 100644 --- a/site.conf +++ b/site.conf @@ -71,6 +71,7 @@ '56c4201f6ce2994678b0142e19099dd28d6ed17775d35ca9a7f12d9235890ffc', -- chris '3cc6dc6d0d7499d4d17cacad5f16336634e0b26700807f29cc241eaabbde481f', -- goligo 'e7a50df4bc8875113eaf75dff5204e8753a9189baeed40c6db0509398a428820', -- thegoliath + '50bafd8216cab2ee1c11c215b528dd7c6396f3edfdab689c70ca04a9f284b931', -- grische }, }, testing = { @@ -92,6 +93,7 @@ '56c4201f6ce2994678b0142e19099dd28d6ed17775d35ca9a7f12d9235890ffc', -- chris '3cc6dc6d0d7499d4d17cacad5f16336634e0b26700807f29cc241eaabbde481f', -- goligo 'e7a50df4bc8875113eaf75dff5204e8753a9189baeed40c6db0509398a428820', -- thegoliath + '50bafd8216cab2ee1c11c215b528dd7c6396f3edfdab689c70ca04a9f284b931', -- grische }, }, experimental = { @@ -113,6 +115,7 @@ '56c4201f6ce2994678b0142e19099dd28d6ed17775d35ca9a7f12d9235890ffc', -- chris '3cc6dc6d0d7499d4d17cacad5f16336634e0b26700807f29cc241eaabbde481f', -- goligo 'e7a50df4bc8875113eaf75dff5204e8753a9189baeed40c6db0509398a428820', -- thegoliath + '50bafd8216cab2ee1c11c215b528dd7c6396f3edfdab689c70ca04a9f284b931', -- grische }, }, }, From f4ead381fde08fbff2d20016a2ae5bacac91852f Mon Sep 17 00:00:00 2001 From: goligo Date: Tue, 25 Oct 2022 11:15:29 +0200 Subject: [PATCH 17/29] Add --tags to git fetch to make sure all Gluon tags are included When preparation the gluon-build folder was changed from git clone to git fetch, the tags got lost. This is causing the generated fIrmware version not to incluce the gluon version correctly: "v2022.10.4 / gluon-ab1fb05+" instead of "v2022.10.4 / gluon-v2022.1.1+" --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b2ea8fcf..3b091549 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,7 @@ ${GLUON_BUILD_DIR}/.git: | ${GLUON_BUILD_DIR} cd ${GLUON_BUILD_DIR} && git remote add origin ${GLUON_GIT_URL} gluon-update: | ${GLUON_BUILD_DIR}/.git - cd ${GLUON_BUILD_DIR} && git fetch origin ${GLUON_GIT_REF} + cd ${GLUON_BUILD_DIR} && git fetch --tags origin ${GLUON_GIT_REF} cd ${GLUON_BUILD_DIR} && git reset --hard FETCH_HEAD cd ${GLUON_BUILD_DIR} && git clean -ffdx packages/ # double -f -f required to remove sub-repositories cd ${GLUON_BUILD_DIR} && git clean -fd From 98eb1aa6dfd1ae8cd5be8c5e8cbd52fc8c931c8d Mon Sep 17 00:00:00 2001 From: Malte Wedel Date: Thu, 27 Oct 2022 17:41:41 +0200 Subject: [PATCH 18/29] Get rid of ::set-ouput as it is deprecated --- .github/workflows/firmware.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/firmware.yml b/.github/workflows/firmware.yml index 49dce3aa..bd0a0f32 100644 --- a/.github/workflows/firmware.yml +++ b/.github/workflows/firmware.yml @@ -23,7 +23,7 @@ jobs: shell: bash run: | target_list="[\"ar71xx-generic\", \"ar71xx-tiny\", \"ar71xx-nand\", \"ath79-generic\", \"brcm2708-bcm2708\", \"brcm2708-bcm2709\", \"ipq40xx-generic\", \"ipq806x-generic\", \"lantiq-xrx200\", \"lantiq-xway\", \"mpc85xx-generic\", \"mpc85xx-p1020\", \"ramips-mt7620\", \"ramips-mt7621\", \"ramips-mt76x8\", \"ramips-rt305x\", \"sunxi-cortexa7\", \"x86-generic\", \"x86-geode\", \"x86-legacy\", \"x86-64\", \"ar71xx-mikrotik\", \"brcm2708-bcm2710\", \"mvebu-cortexa9\"]" - echo ::set-output name=target::{\"target\": $(echo $target_list)}\" + echo "target={\"target\": $(echo $target_list)}" >> $GITHUB_OUTPUT build_firmware: needs: generate_target_matrix @@ -44,7 +44,7 @@ jobs: run: | git checkout -b patched ${GITHUB_SHA} make BROKEN=1 GLUON_TARGETS=${{ matrix.target }} V=s - echo "::set-output name=status::success" + echo "status=success" >> $GITHUB_OUTPUT - name: Upload firmware ${{ matrix.target }} uses: actions/upload-artifact@master if: steps.compile.outputs.status == 'success' From d10851adb84a30069188d295e9e0515c51b200d9 Mon Sep 17 00:00:00 2001 From: goligo Date: Thu, 27 Oct 2022 14:04:29 +0200 Subject: [PATCH 19/29] Build without easimon/maximize-build-space Due to stability issues with GitHub runners, try to run without LVM magic, instead just remove some big software packages. --- .github/workflows/firmware.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/firmware.yml b/.github/workflows/firmware.yml index bd0a0f32..6c053182 100644 --- a/.github/workflows/firmware.yml +++ b/.github/workflows/firmware.yml @@ -32,6 +32,11 @@ jobs: matrix: ${{ fromJson(needs.generate_target_matrix.outputs.target_json) }} runs-on: ubuntu-latest steps: + - name: Maximize build space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc - name: Checkout uses: actions/checkout@master with: From d230408a6ec37ae9958e6f9015000599be8387b5 Mon Sep 17 00:00:00 2001 From: Christian Date: Sun, 5 Feb 2023 13:09:12 +0100 Subject: [PATCH 20/29] Add sign.sh from Gluon File from https://github.com/freifunk-gluon/gluon/blob/master/contrib/sign.sh --- .github/workflows/firmware.yml | 1 + contrib/sign.sh | 80 ++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100755 contrib/sign.sh diff --git a/.github/workflows/firmware.yml b/.github/workflows/firmware.yml index 6c053182..4dec15b3 100644 --- a/.github/workflows/firmware.yml +++ b/.github/workflows/firmware.yml @@ -11,6 +11,7 @@ on: - '**/CODEOWNERS' - '.github/ISSUE_TEMPLATE' - '.github/*.yml' + - 'contrib/sign.sh' jobs: generate_target_matrix: diff --git a/contrib/sign.sh b/contrib/sign.sh new file mode 100755 index 00000000..3bc794c8 --- /dev/null +++ b/contrib/sign.sh @@ -0,0 +1,80 @@ +#!/bin/sh + +# Copyright (c) Project Gluon +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# SPDX-License-Identifier: BSD-2-Clause + +set -e + +if [ $# -ne 2 ] || [ "-h" = "$1" ] || [ "--help" = "$1" ] || [ ! -r "$1" ] || [ ! -r "$2" ]; then + cat < + +sign.sh adds lines to a manifest to indicate the approval +of the integrity of the firmware as required for automated +updates. The first argument references a file harboring +the private key of a public-private key pair of a developer +that referenced by its public key in the site configuration. +The script may be performed multiple times to the same document +to indicate an approval by multiple developers. + +See also + * ecdsautils on https://github.com/freifunk-gluon/ecdsautils + +EOHELP + exit 1 +fi + +SECRET="$1" + +manifest="$2" +upper="$(mktemp)" +lower="$(mktemp)" + +trap 'rm -f "$upper" "$lower"' EXIT + +awk 'BEGIN { + sep = 0 +} + +/^---$/ { + sep = 1; + next +} + +{ + if(sep == 0) { + print > "'"$upper"'" + } else { + print > "'"$lower"'" + } +}' "$manifest" + +ecdsasign "$upper" < "$SECRET" >> "$lower" + +( + cat "$upper" + echo --- + cat "$lower" +) > "$manifest" From c51034f19f29aaefe6e9fe28d4adef585d7feb11 Mon Sep 17 00:00:00 2001 From: Malte Wedel Date: Wed, 18 Jan 2023 23:27:11 +0100 Subject: [PATCH 21/29] Put target list into a separate file As the Github build is building targets in separate VMs, it does not use the target-list in the Makefile, but holds its own copy instead. This change moves the target list into a new file called "targets" which is then used by the Makefile as well as the Github workflow. --- .github/workflows/firmware.yml | 4 +++- Makefile | 27 +-------------------------- targets | 24 ++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 targets diff --git a/.github/workflows/firmware.yml b/.github/workflows/firmware.yml index 4dec15b3..898ced70 100644 --- a/.github/workflows/firmware.yml +++ b/.github/workflows/firmware.yml @@ -19,11 +19,13 @@ jobs: outputs: target_json: ${{ steps.set_target.outputs.target }} steps: + - name: Checkout + uses: actions/checkout@v3 - name: Set target matrix id: set_target shell: bash run: | - target_list="[\"ar71xx-generic\", \"ar71xx-tiny\", \"ar71xx-nand\", \"ath79-generic\", \"brcm2708-bcm2708\", \"brcm2708-bcm2709\", \"ipq40xx-generic\", \"ipq806x-generic\", \"lantiq-xrx200\", \"lantiq-xway\", \"mpc85xx-generic\", \"mpc85xx-p1020\", \"ramips-mt7620\", \"ramips-mt7621\", \"ramips-mt76x8\", \"ramips-rt305x\", \"sunxi-cortexa7\", \"x86-generic\", \"x86-geode\", \"x86-legacy\", \"x86-64\", \"ar71xx-mikrotik\", \"brcm2708-bcm2710\", \"mvebu-cortexa9\"]" + target_list=$(cat targets | jq -Rsc '.[:-1] | split("\n")') echo "target={\"target\": $(echo $target_list)}" >> $GITHUB_OUTPUT build_firmware: diff --git a/Makefile b/Makefile index 3b091549..e2269c7a 100644 --- a/Makefile +++ b/Makefile @@ -5,32 +5,7 @@ GLUON_GIT_REF := v2021.1.2 PATCH_DIR := ./patches SECRET_KEY_FILE ?= ${HOME}/.gluon-secret-key -GLUON_TARGETS ?= \ - ar71xx-generic \ - ar71xx-mikrotik \ - ar71xx-nand \ - ar71xx-tiny \ - ath79-generic \ - brcm2708-bcm2708 \ - brcm2708-bcm2709 \ - brcm2708-bcm2710 \ - ipq40xx-generic \ - ipq806x-generic \ - lantiq-xrx200 \ - lantiq-xway \ - mpc85xx-generic \ - mpc85cc-p1020 \ - mvebu-cortexa9 \ - ramips-mt7620 \ - ramips-mt7621 \ - ramips-mt76x8 \ - ramips-rt305x \ - sunxi-cortexa7 \ - x86-64 \ - x86-generic \ - x86-geode \ - x86-legacy - +GLUON_TARGETS ?= $(shell cat targets | tr '\n' ' ') GLUON_AUTOUPDATER_BRANCH := stable ifneq (,$(shell git describe --exact-match --tags 2>/dev/null)) diff --git a/targets b/targets new file mode 100644 index 00000000..d1c65d1e --- /dev/null +++ b/targets @@ -0,0 +1,24 @@ +ar71xx-generic +ar71xx-mikrotik +ar71xx-nand +ar71xx-tiny +ath79-generic +brcm2708-bcm2708 +brcm2708-bcm2709 +brcm2708-bcm2710 +ipq40xx-generic +ipq806x-generic +lantiq-xrx200 +lantiq-xway +mpc85xx-generic +mpc85xx-p1020 +mvebu-cortexa9 +ramips-mt7620 +ramips-mt7621 +ramips-mt76x8 +ramips-rt305x +sunxi-cortexa7 +x86-64 +x86-generic +x86-geode +x86-legacy From ca37c41c561f509c356959ea976ed1cf580e19e5 Mon Sep 17 00:00:00 2001 From: Christian Date: Sun, 5 Feb 2023 12:16:44 +0100 Subject: [PATCH 22/29] Replace deprecated actions/create-release --- .github/workflows/firmware.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/firmware.yml b/.github/workflows/firmware.yml index 898ced70..84c1c203 100644 --- a/.github/workflows/firmware.yml +++ b/.github/workflows/firmware.yml @@ -69,14 +69,13 @@ jobs: steps: - name: Create Release id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: softprops/action-gh-release@v1 with: tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} + name: Release ${{ github.ref }} draft: false prerelease: false + generate_release_notes: true upload_release: strategy: From c194ccc3792cd251816e850b799e32c9a08aea0b Mon Sep 17 00:00:00 2001 From: Christian Date: Sun, 5 Feb 2023 12:24:51 +0100 Subject: [PATCH 23/29] Update actions/download-artifact for download-all --- .github/workflows/firmware.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/firmware.yml b/.github/workflows/firmware.yml index 84c1c203..2cf3f4e1 100644 --- a/.github/workflows/firmware.yml +++ b/.github/workflows/firmware.yml @@ -87,11 +87,10 @@ jobs: - generate_target_matrix if: github.event_name != 'pull_request' steps: - - name: Download Artifact ${{ matrix.target }} - uses: actions/download-artifact@v2 - with: - name: ${{ matrix.target }}_output - path: ${{ matrix.target }}_output + - name: Download Artifacts for ${{ matrix.target }} + uses: actions/download-artifact@v3 + - name: Display structure of artifacts + run: ls -R - name: Create ${{ matrix.target }}_output.tar.gz run: tar zcvf ${{ matrix.target }}_output.tar.gz ${{ matrix.target }}_output - name: Upload Release Asset ${{ matrix.target }} From acbf1a6a9ba7f6dbb92f76bf35dd653f2651255c Mon Sep 17 00:00:00 2001 From: Christian Date: Sun, 5 Feb 2023 12:27:39 +0100 Subject: [PATCH 24/29] Simplifying Release Creation --- .github/workflows/firmware.yml | 50 ++++++++++++---------------------- 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/.github/workflows/firmware.yml b/.github/workflows/firmware.yml index 2cf3f4e1..2c786d79 100644 --- a/.github/workflows/firmware.yml +++ b/.github/workflows/firmware.yml @@ -64,42 +64,26 @@ jobs: runs-on: ubuntu-latest needs: build_firmware if: github.event_name != 'pull_request' - outputs: - output1: ${{ steps.create_release.outputs.upload_url }} - steps: - - name: Create Release - id: create_release - uses: softprops/action-gh-release@v1 - with: - tag_name: ${{ github.ref }} - name: Release ${{ github.ref }} - draft: false - prerelease: false - generate_release_notes: true - - upload_release: - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.generate_target_matrix.outputs.target_json) }} - runs-on: ubuntu-latest - needs: - - create_release - - generate_target_matrix - if: github.event_name != 'pull_request' steps: - name: Download Artifacts for ${{ matrix.target }} uses: actions/download-artifact@v3 - name: Display structure of artifacts run: ls -R - - name: Create ${{ matrix.target }}_output.tar.gz - run: tar zcvf ${{ matrix.target }}_output.tar.gz ${{ matrix.target }}_output - - name: Upload Release Asset ${{ matrix.target }} - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Create tar.gz files + run: | + set -x + for output in *_output; do + tar zcvf "${output}.tar.gz" "${output}" + done + - name: Create Release & Upload Release Assets + uses: softprops/action-gh-release@v1 with: - upload_url: ${{ needs.create_release.outputs.output1 }} - asset_path: ./${{ matrix.target }}_output.tar.gz - asset_name: ${{ matrix.target }}_output.tar.gz - asset_content_type: application/gzip + # Note: If there is no release name specified, releases created in + # the GitHub UI do not trigger a failure and are modified instead. + draft: false + prerelease: false + # Note: Release notes are only auto-generated if the release was + # created by the Github Action and was not created in the Github UI. + generate_release_notes: true + files: | + ./*_output.tar.gz From bf0d5f7395f8ccbfd6578ee18edd0a28c2dfea26 Mon Sep 17 00:00:00 2001 From: Christian Date: Sun, 7 May 2023 22:58:36 +0200 Subject: [PATCH 25/29] fix name for download-artifact action Follow-up of 488c430e7031a9e25c1b3864bc42820e0fc3ed96 --- .github/workflows/firmware.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/firmware.yml b/.github/workflows/firmware.yml index 2c786d79..1c66f32c 100644 --- a/.github/workflows/firmware.yml +++ b/.github/workflows/firmware.yml @@ -65,7 +65,7 @@ jobs: needs: build_firmware if: github.event_name != 'pull_request' steps: - - name: Download Artifacts for ${{ matrix.target }} + - name: Download all artifacts uses: actions/download-artifact@v3 - name: Display structure of artifacts run: ls -R From 8f552c2e4cf9a459e6e6956131a8b34a5c63bc1d Mon Sep 17 00:00:00 2001 From: Grische Date: Fri, 21 Jul 2023 15:52:56 +0200 Subject: [PATCH 26/29] github: update action-shellcheck to v1.18.1 --- .github/workflows/shellcheck.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 8744f86e..6ce5ac46 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -1,6 +1,7 @@ --- name: Shell Linting -on: + +on: # yamllint disable-line rule:truthy workflow_dispatch: push: branches: @@ -21,7 +22,7 @@ jobs: - name: shellcheck # Make sure the action is pinned to a commit, as all reviewdog repos # have hundreds of contributors with write access (breaks easy/often) - uses: reviewdog/action-shellcheck@66c9a47bf02255b250284a82251cb4cadf5043f5 # v1.15.0 + uses: reviewdog/action-shellcheck@50e5e1e2284575f23a1e409d9c0804cdfc4f6e31 # v1.18.1 with: filter_mode: "file" fail_on_error: true From 01b788ebc387ebec839854a41629bcee929ec386 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 26 Jul 2023 18:46:04 +0200 Subject: [PATCH 27/29] changelog: add reference to Github releases --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63582f9a..f991f306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Freifunk München Firmware Changelog +## Newer versions + Detailed changelog for versions v2022.5.1 and above can be found here: + https://github.com/freifunkMUC/site-ffm/releases + ## v2022.2.1 - Fix bug in wireguard package which was causing high CPU load if no WAN was available From a99676f367016de55756a32e23a8f19609e0df8c Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 26 Jul 2023 19:24:41 +0200 Subject: [PATCH 28/29] contrib: make genpkglist executable --- contrib/genpkglist.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 contrib/genpkglist.py diff --git a/contrib/genpkglist.py b/contrib/genpkglist.py old mode 100644 new mode 100755 From 398acfd3572346d17170d71ae3909c6022027c49 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 26 Jul 2023 19:40:13 +0200 Subject: [PATCH 29/29] readme: improve Docker instructions --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f40c6ebc..1d0404db 100644 --- a/README.md +++ b/README.md @@ -25,20 +25,21 @@ make As the CI is using Ubuntu, only the Ubuntu dependencies are being tracked. To simplify building on other distros, containerised building is also possible: ```sh -docker build -t site-ffm -f Dockerfile_build . +docker build -t site-ffm-stable -f Dockerfile_build . ``` This will build the build Docker image. With the following export, the Makefile will then use the repo for building but will run inside an Ubuntu container.\ **Note**: If the working directory is a git worktree, add a volume mount for the main git folder. ```sh -docker run --rm -v $(pwd):/site-ffm:ro -v $(pwd)/gluon-build:/site-ffm/gluon-build:rw -v $(pwd)/output:/site-ffm/output:rw -w /site-ffm -u $UID site-ffm-next make +mkdir -p gluon-build output +docker run --rm -v $(pwd):/site-ffm:ro -v $(pwd)/gluon-build:/site-ffm/gluon-build:rw -v $(pwd)/output:/site-ffm/output:rw -w /site-ffm -u $UID site-ffm-stable make ``` #### Example Full command for a [lantiq-xrx200](https://github.com/freifunk-gluon/gluon/blob/v2022.1/targets/lantiq-xrx200) build: ```sh -mkdir -p logs -docker run --rm -v $(pwd):/site-ffm:ro -v $(pwd)/gluon-build:/site-ffm/gluon-build:rw -v $(pwd)/output:/site-ffm/output:rw -w /site-ffm -u $UID site-ffm-next make V=s BROKEN=1 GLUON_TARGETS=lantiq-xrx200 |& tee logs/buildtest_lantiq-xrx200_$(date --iso=s).log +mkdir -p logs gluon-build output +docker run --rm -v $(pwd):/site-ffm:ro -v $(pwd)/gluon-build:/site-ffm/gluon-build:rw -v $(pwd)/output:/site-ffm/output:rw -w /site-ffm -u $UID site-ffm-stable make V=s GLUON_TARGETS=lantiq-xrx200 |& tee logs/build_lantiq-xrx200_$(date --iso=s).log ``` ## Further Resources