Update to Kubo v0.23.0 - add prebuilt-binary pipeline

.github/workflows/release.yml

@@ -0,0 +1,120 @@
+name: Kubo plugin
+
+on:
+  push:
+
+
+env:
+  # must be same as official ipfs builds. See distributions/.tool-versions
+  GO_VERSION: "1.21.1"
+
+jobs:
+  build-artifacts:
+    name: "Build"
+    strategy:
+      matrix:
+        os:
+          - ubuntu-20.04
+          - macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 2
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: ${{ env.GO_VERSION }}
+    - name: Extract release name from tag
+      run: |
+        RELEASE=$(basename ${{ github.ref_name }})
+        echo "RELEASE=$RELEASE" >> "$GITHUB_ENV"
+    - name: Package plugin
+      run: make dist-plugin
+    - name: Record Go environment
+      run: |
+        echo "GOHOSTOS=$(go env GOHOSTOS)" >> "$GITHUB_ENV"
+        echo "GOHOSTARCH=$(go env GOHOSTARCH)" >> "$GITHUB_ENV"
+    - name: Rename package
+      run: |
+        NAME="go-ds-s3-plugin_${{ env.RELEASE }}_${{ env.GOHOSTOS }}_${{ env.GOHOSTARCH }}.tar.gz"
+        mv go-ds-s3-plugin/go-ds-s3-plugin.tar.gz "$NAME"
+        echo "ARTIFACT_NAME=$NAME" >> "$GITHUB_ENV"
+    - name: Archive artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ env.ARTIFACT_NAME }}
+        path: go-ds-s3-plugin_*.tar.gz
+
+  test-artifacts:
+    name: "Test"
+    needs: build-artifacts
+    strategy:
+      matrix:
+        os:
+          - ubuntu-20.04
+# macos test fail with dist-built ipfs.
+#          - macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 2
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: ${{ env.GO_VERSION }}
+    - name: Record variables
+      run: |
+        V=`cat go.mod | grep github.com/ipfs/kubo | grep -o 'v.*'`
+        echo "KUBO_VERSION=$V" >> "$GITHUB_ENV"
+        GOHOSTOS=`go env GOHOSTOS`
+        echo "GOHOSTOS=$GOHOSTOS" >> "$GITHUB_ENV"
+        RELEASE=$(basename ${{ github.ref_name }})
+        echo "RELEASE=$RELEASE" >> "$GITHUB_ENV"
+        echo PLUGIN_ARTIFACT=go-ds-s3-plugin_${RELEASE}_${GOHOSTOS}_amd64.tar.gz >> "$GITHUB_ENV"
+    - name: Download artifact
+      id: download
+      uses: actions/download-artifact@v3
+      with:
+        name: ${{ env.PLUGIN_ARTIFACT }}
+    - name: Download unpack Kubo
+      run: |
+        wget -nv https://dist.ipfs.tech/kubo/${{ env.KUBO_VERSION }}/kubo_${{ env.KUBO_VERSION }}_${{ env.GOHOSTOS }}-amd64.tar.gz
+        tar -xf kubo_${{ env.KUBO_VERSION }}_${{ env.GOHOSTOS }}-amd64.tar.gz
+        tar -xf go-ds-s3-plugin_*.tar.gz
+        chmod +x kubo/ipfs
+    - name: Initialize IPFS and copy plugin
+      run: |
+        export IPFS_PATH=$(pwd)/ipfs-config
+        echo "IPFS_PATH=$IPFS_PATH" >> "$GITHUB_ENV"
+        ./kubo/ipfs init
+        mkdir -p ipfs-config/plugins
+        cp go-ds-s3-plugin/go-ds-s3-plugin ipfs-config/plugins/
+    - name: Check IPFS works with the plugin
+      run: ./kubo/ipfs version --all
+
+  release:
+    name: "Release"
+    needs: test-artifacts
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Download artifacts
+      id: download
+      uses: actions/download-artifact@v3
+    - name: Extract release name from tag
+      run: |
+        RELEASE=$(basename ${{ github.ref_name }})
+        echo "RELEASE=$RELEASE" >> "$GITHUB_ENV"
+    - name: Release
+      uses: softprops/action-gh-release@v1
+      if: startsWith(github.ref, 'refs/tags/go-ds-s3-plugin/v')
+      with:
+        files: go-ds-s3-plugin_*.tar.gz/*
+        body: |
+          This is a binary build of the go-ds-s3 Kubo plugin targeting Kubo version ${{ env.RELEASE }}.
+
+          To install, download the relevant asset for your platform, unpack the plugin file (`go-ds-s3-plugin`) and place it in `~/.ipfs/plugins`. MacOS users will need to compile Kubo themselves, as the official releases have no CGO support.
+
+          See the included README.md for more information.
+        name: ${{ github.ref_name }}

Makefile

@@ -1,38 +1,15 @@
-# Force Go Modules
-GO111MODULE = on
+plugin:
+	$(MAKE) -C go-ds-s3-plugin all
 
-GOCC ?= go
-GOFLAGS ?=
+install-plugin:
+	$(MAKE) -C go-ds-s3-plugin install
 
-# If set, override the install location for plugins
-IPFS_PATH ?= $(HOME)/.ipfs
+dist-plugin:
+	$(MAKE) -C go-ds-s3-plugin dist
 
-# If set, override the IPFS version to build against. This _modifies_ the local
-# go.mod/go.sum files and permanently sets this version.
-IPFS_VERSION ?= $(lastword $(shell $(GOCC) list -m github.com/ipfs/kubo))
+check:
+	go vet ./...
+	staticcheck --checks all ./...
+	misspell -error -locale US .
 
-# make reproducible
-ifneq ($(findstring /,$(IPFS_VERSION)),)
-# Locally built kubo
-GOFLAGS += -asmflags=all=-trimpath="$(GOPATH)" -gcflags=all=-trimpath="$(GOPATH)"
-else
-# Remote version of kubo (e.g. via `go get -trimpath` or official distribution)
-GOFLAGS += -trimpath
-endif
-
-.PHONY: install build
-
-go.mod: FORCE
-	./set-target.sh $(IPFS_VERSION)
-
-FORCE:
-
-s3plugin.so: plugin/main/main.go go.mod
-	CGO_ENABLED=1 $(GOCC) build $(GOFLAGS) -buildmode=plugin -o "$@" "$<"
-	chmod +x "$@"
-
-build: s3plugin.so
-	@echo "Built against" $(IPFS_VERSION)
-
-install: build
-	install -Dm700 s3plugin.so "$(IPFS_PATH)/plugins/go-ds-s3.so"
+.PHONY: plugin install-plugin check

README.md

@@ -4,16 +4,38 @@ This is an implementation of the datastore interface backed by amazon s3.
 
 **NOTE:** Plugins only work on Linux and MacOS at the moment. You can track the progress of this issue here: https://github.com/golang/go/issues/19282
 
-## Building and Installing
+## Quickstart
 
-You must build the plugin with the *exact* version of go used to build the kubo binary you will use it with. You can find the go version for kubo builds from dist.ipfs.io in the build-info file, e.g. https://dist.ipfs.io/kubo/v0.4.22/build-info or by running `ipfs version --all`.
+  1. Grab a plugin release from the [releases](https://github.com/ipfs/go-ds-s3/releases) section matching your Kubo version and install the plugin file in `~/.ipfs/plugins`.
+  2. Follow the instructions in the plugin's [README.md](go-ds-s3-plugin/README.md)
 
-In addition to needing the exact version of go, you need to build the correct version of this plugin.
 
-* To build against a released version of kubo, checkout the `release/v$VERSION` branch and build.
-* To build against a custom (local) build of kubo, run `make IPFS_VERSION=/path/to/kubo/source`.
+## Building and installing
 
-You can then install it into your local IPFS repo by running `make install`.
+
+The plugin can be manually built/installed for different versions of Kubo (starting with 0.23.0) with:
+
+```
+git checkout go-ds-s3-plugin/v<kubo-version>
+make plugin
+make install-plugin
+```
+
+## Updating to a new version
+
+  1. `go get` the Kubo release you want to build for. Make sure any other
+     dependencies are aligned to what Kubo uses.
+  2. `make install` and test.
+
+
+If you are building against dist-released versions of Kubo, you need to build using the same version of go that was used to build the release ([here](https://github.com/ipfs/distributions/blob/master/.tool-versions)).
+
+If you are building against your own build of Kubo you must align your plugin to use it.
+
+If you are updating this repo to produce a new version of the plugin:
+
+  1. Submit a PR so that integration tests run
+  2. Make a new tag `go-ds-s3-plugin/v<kubo_version>` and push it. This will build and release the plugin prebuilt binaries.
 
 ## Bundling
 
@@ -46,74 +68,6 @@ As go plugins can be finicky to correctly compile and install, you may want to c
 > make install
 ```
 
-## Detailed Installation
-
-For a brand new ipfs instance (no data stored yet):
-
-1. Copy s3plugin.so $IPFS_DIR/plugins/go-ds-s3.so (or run `make install` if you are installing locally).
-2. Run `ipfs init`.
-3. Edit $IPFS_DIR/config to include s3 details (see Configuration below).
-4. Overwrite `$IPFS_DIR/datastore_spec` as specified below (*Don't do this on an instance with existing data - it will be lost*).
-
-### Configuration
-
-The config file should include the following:
-```json
-{
-  "Datastore": {
-  ...
-
-    "Spec": {
-      "mounts": [
-        {
-          "child": {
-            "type": "s3ds",
-            "region": "us-east-1",
-            "bucket": "$bucketname",
-            "rootDirectory": "$bucketsubdirectory",
-            "accessKey": "",
-            "secretKey": ""
-          },
-          "mountpoint": "/blocks",
-          "prefix": "s3.datastore",
-          "type": "measure"
-        },
-```
-
-If the access and secret key are blank they will be loaded from the usual ~/.aws/.
-If you are on another S3 compatible provider, e.g. Linode, then your config should be:
-
-```json
-{
-  "Datastore": {
-  ...
-
-    "Spec": {
-      "mounts": [
-        {
-          "child": {
-            "type": "s3ds",
-            "region": "us-east-1",
-            "bucket": "$bucketname",
-            "rootDirectory": "$bucketsubdirectory",
-            "regionEndpoint": "us-east-1.linodeobjects.com",
-            "accessKey": "",
-            "secretKey": ""
-          },
-          "mountpoint": "/blocks",
-          "prefix": "s3.datastore",
-          "type": "measure"
-        },
-```
-
-If you are configuring a brand new ipfs instance without any data, you can overwrite the datastore_spec file with:
-
-```
-{"mounts":[{"bucket":"$bucketname","mountpoint":"/blocks","region":"us-east-1","rootDirectory":"$bucketsubdirectory"},{"mountpoint":"/","path":"datastore","type":"levelds"}],"type":"mount"}
-```
-
-Otherwise, you need to do a datastore migration.
-
 ## Contribute
 
 Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-example-plugin/issues)!

go-ds-s3-plugin/Makefile

@@ -0,0 +1,23 @@
+SRC := $(shell find ../.. -type f -name '*.go')
+IPFS_PATH ?= $(HOME)/.ipfs
+
+export CGO_ENABLED := 1
+
+all: build
+
+build: $(SRC)
+	go build -buildmode=plugin -trimpath -o go-ds-s3-plugin
+
+install: build
+	mkdir -p "$(IPFS_PATH)/.ipfs/plugins"
+	install -Dm700 go-ds-s3-plugin "$(IPFS_PATH)/plugins/go-ds-s3-plugin"
+
+dist: build
+	mkdir -p dist/go-ds-s3-plugin
+	cp go-ds-s3-plugin README.md dist/go-ds-s3-plugin/
+	chmod +x dist/go-ds-s3-plugin/go-ds-s3-plugin
+	tar -C dist -zcf go-ds-s3-plugin.tar.gz go-ds-s3-plugin
+	rm -rf go-ds-s3-plugin
+	echo "Packaged as go-ds-s3-plugin.tar.gz"
+
+.PHONY: install