Skip to content

Commit

Permalink
try accessing Backblaze
Browse files Browse the repository at this point in the history
  • Loading branch information
@marten-seemann
marten-seemann committed Oct 6, 2023
1 parent 44ea912 commit 1c0bc2d
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 355 deletions.
293 changes: 13 additions & 280 deletions .github/workflows/interop.yml
View file
Original file line number Diff line number Diff line change
@@ -1,293 +1,26 @@
name: interop
on:
schedule:
- cron: "0 */8 * * *" # every 8h

# Cache key for caching the Wireshark build.
# To trigger a rebuild of Wireshark increment this value.
# The rebuild will then build the current master of Wireshark and save it under the new key.
env:
WIRESHARK_CACHEKEY: 6
push:
branches: ["s3-setup"]

jobs:
wireshark:
runs-on: ubuntu-latest
steps:
- name: Restore from cache
id: restore-cache
uses: actions/cache@v3
env:
VERSION: ${{ env.WIRESHARK_CACHEKEY }}
with:
key: wireshark-${{ env.VERSION }}
path: tshark.tar.gz
- name: Show tshark version information
if: steps.restore-cache.outputs.cache-hit == 'true'
run: |
tar xfz tshark.tar.gz
./tshark -v
- uses: actions/checkout@v3
if: steps.restore-cache.outputs.cache-hit != 'true'
with:
repository: the-tcpdump-group/libpcap
- name: Build libpcap
if: steps.restore-cache.outputs.cache-hit != 'true'
run: |
./autogen.sh
./configure --disable-dbus --disable-rdma
sudo make install
- uses: actions/checkout@v3
if: steps.restore-cache.outputs.cache-hit != 'true'
with:
repository: wireshark/wireshark
- name: Install dependencies
if: steps.restore-cache.outputs.cache-hit != 'true'
run: sudo apt-get install -y cmake libglib2.0-dev libc-ares-dev libgcrypt20-dev flex bison byacc ninja-build
- name: Build Wireshark
if: steps.restore-cache.outputs.cache-hit != 'true'
run: |
cmake -GNinja -DBUILD_wireshark=0 -DBUILD_qtshark=0 -DBUILD_editcap=0 -DBUILD_capinfos=0 -DBUILD_text2pcap=0 -DBUILD_rawshark=0 -DBUILD_sdjournal=0 -DBUILD_sshdump=0 -DBUILD_ciscodump=0 -DBUILD_sharkd=0 -DENABLE_STATIC=1 -DENABLE_PLUGINS=0 -DENABLE_LIBXML2=0 -DENABLE_BROTLI=0 -DUSE_STATIC=1 -DENABLE_GNUTLS=1 .
ninja
- run: run/tshark -v
if: steps.restore-cache.outputs.cache-hit != 'true'
- name: Compress
if: steps.restore-cache.outputs.cache-hit != 'true'
run: tar -czvf tshark.tar.gz -C run/ tshark
- name: Upload
uses: actions/upload-artifact@v3
with:
name: wireshark
path: tshark.tar.gz
config:
runs-on: ubuntu-latest
outputs:
logname: ${{ steps.set-logname.outputs.logname }}
starttime: ${{ steps.set-starttime.outputs.starttime }}
servers: ${{ steps.set-servers.outputs.servers }}
clients: ${{ steps.set-clients.outputs.clients }}
images: ${{ steps.set-images.outputs.images }}
steps:
- name: Set log name
id: set-logname
run: |
LOGNAME=$(date -u +"%Y-%m-%dT%H:%M")
echo $LOGNAME
echo "logname=$LOGNAME" >> $GITHUB_OUTPUT
- name: Save start time
id: set-starttime
run: |
STARTTIME=$(date +%s)
echo $STARTTIME
echo "starttime=$STARTTIME" >> $GITHUB_OUTPUT
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Determine servers
id: set-servers
run: |
SERVERS=$(jq -c 'with_entries(select(.value.role == "server" or .value.role == "both")) | keys_unsorted' implementations.json)
echo $SERVERS
echo "servers=$SERVERS" >> $GITHUB_OUTPUT
- name: Determine clients
id: set-clients
run: |
CLIENTS=$(jq -c 'with_entries(select(.value.role == "client" or .value.role == "both")) | keys_unsorted' implementations.json)
echo $CLIENTS
echo "clients=$CLIENTS" >> $GITHUB_OUTPUT
- name: Determine Docker images
id: set-images
run: |
IMAGES=$(jq -c 'keys_unsorted' implementations.json)
echo $IMAGES
echo "images=$IMAGES" >> $GITHUB_OUTPUT
docker-pull-tools:
runs-on: ubuntu-latest
strategy:
matrix:
image: [ 'quic-network-simulator', 'quic-interop-iperf-endpoint' ]
steps:
- uses: actions/checkout@v3
- name: Pull
run: |
URL="martenseemann/${{ matrix.image }}"
docker pull $URL
echo "URL=$URL" >> $GITHUB_ENV
- name: Docker inspect
run: docker image inspect $URL
- name: Save Docker image
run: |
docker save $URL | gzip --best > ${{ matrix.image }}.tar.gz
du -sh ${{ matrix.image }}.tar.gz
- name: Upload result
uses: actions/upload-artifact@v3
with:
name: images-tools
path: ${{ matrix.image }}.tar.gz
if-no-files-found: error
docker-pull-images:
needs: [ config ]
runs-on: ubuntu-latest
strategy:
matrix:
image: ${{ fromJson(needs.config.outputs.images) }}
name: Pull ${{ matrix.image }}
steps:
- uses: actions/checkout@v3
- name: Run docker pull
run: |
URL=$(jq -r '.["${{ matrix.image }}"].image' implementations.json)
echo $URL
docker pull $URL
echo "URL=$URL" >> $GITHUB_ENV
- name: Docker inspect
run: docker image inspect $URL
- name: Save Docker image
run: |
docker save $URL | gzip --best > ${{ matrix.image }}.tar.gz
du -sh ${{ matrix.image }}.tar.gz
- name: Upload result
uses: actions/upload-artifact@v3
with:
name: image-${{ matrix.image }}
path: ${{ matrix.image }}.tar.gz
if-no-files-found: error
tests:
needs: [ wireshark, config, docker-pull-tools, docker-pull-images ]
runs-on: ubuntu-latest
continue-on-error: true
strategy:
fail-fast: false
matrix:
server: ${{ fromJson(needs.config.outputs.servers) }}
client: ${{ fromJson(needs.config.outputs.clients) }}
name: (${{ matrix.server }} - ${{ matrix.client }})
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Enable IPv6 support
run: sudo modprobe ip6table_filter
- run: docker image ls
- name: Download tools Docker images
uses: actions/download-artifact@v3
with:
name: images-tools
- name: Download ${{ matrix.server }} Docker image
uses: actions/download-artifact@v3
with:
name: image-${{ matrix.server }}
- name: Download ${{ matrix.client }} Docker image
if: ${{ matrix.server != matrix.client }}
uses: actions/download-artifact@v3
with:
name: image-${{ matrix.client }}
- name: Load docker images
run: |
docker load --input quic-network-simulator.tar.gz
docker load --input quic-interop-iperf-endpoint.tar.gz
docker load --input ${{ matrix.server }}.tar.gz
docker load --input ${{ matrix.client }}.tar.gz
- run: docker image ls
- name: Download Wireshark
uses: actions/download-artifact@v3
with:
name: wireshark
path: wireshark
- name: Install Wireshark
run: |
cd wireshark
tar xfz tshark.tar.gz
sudo mv tshark /usr/local/bin
cd .. && rm -r wireshark
- name: Install Python packages
run: |
pip install -U pip
pip install -r requirements.txt
- name: Run tests
env:
CRON: "true"
run: |
(python run.py --client ${{ matrix.client }} --server ${{ matrix.server }} --log-dir logs --json ${{ matrix.server }}_${{ matrix.client }}_results.json -t onlyTests || true) | tee output.txt
mkdir -p logs/${{ matrix.server }}_${{ matrix.client }}
mv output.txt logs/${{ matrix.server }}_${{ matrix.client }}/
- name: Run measurements
env:
CRON: "true"
run: |
python run.py --client ${{ matrix.client }} --server ${{ matrix.server }} --log-dir logs_measurement --json ${{ matrix.server }}_${{ matrix.client }}_measurements.json -t onlyMeasurements || true
if [ ! -d "logs_measurement" ]; then exit 0; fi
find logs_measurement -depth -name "sim" -type d -exec rm -r "{}" \;
find logs_measurement -depth -name "client" -type d -exec rm -r "{}" \;
find logs_measurement -depth -name "server" -type d -exec rm -r "{}" \;
mv logs_measurement/${{ matrix.server }}_${{ matrix.client }}/* logs/${{ matrix.server }}_${{ matrix.client }}/
- name: Upload logs
if: ${{ github.event_name == 'schedule' }}
uses: appleboy/scp-action@master
with:
host: interop.seemann.io
username: ${{ secrets.INTEROP_SEEMANN_IO_USER }}
key: ${{ secrets.INTEROP_SEEMANN_IO_SSH_KEY }}
source: logs/${{ matrix.server }}_${{ matrix.client }}
target: /mnt/logs/${{ needs.config.outputs.logname }}
strip_components: 1
- name: Upload result
uses: actions/upload-artifact@v3
with:
name: results
if-no-files-found: error
path: |
${{ matrix.server }}_${{ matrix.client }}_results.json
${{ matrix.server }}_${{ matrix.client }}_measurements.json
aggregate:
needs: [ config, tests ]
backblaze:
runs-on: ubuntu-latest
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
permissions:
id-token: write
contents: read
env:
LOGNAME: ${{ needs.config.outputs.logname }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Download results
uses: actions/download-artifact@v3
with:
name: results
- name: Aggregate results
run: |
python .github/workflows/aggregate.py \
--start-time ${{ needs.config.outputs.starttime }} \
--server ${{ join(fromJson(needs.config.outputs.servers), ',') }} \
--client ${{ join(fromJson(needs.config.outputs.clients), ',') }} \
--log-dir=$LOGNAME \
--output result.json
- name: Print result
run: jq '.' result.json
- name: Upload result to artifacts
uses: actions/upload-artifact@v3
with:
name: results
path: result.json
- name: Upload result to interop.seemann.io
if: ${{ github.event_name == 'schedule' }}
uses: appleboy/scp-action@master
with:
host: interop.seemann.io
username: ${{ secrets.INTEROP_SEEMANN_IO_USER }}
key: ${{ secrets.INTEROP_SEEMANN_IO_SSH_KEY }}
source: result.json
target: /mnt/logs/${{ needs.config.outputs.logname }}
- name: Publish result
if: ${{ github.event_name == 'schedule' }}
uses: appleboy/ssh-action@master
- run: cat "foobar" > result.json
- name: Configure S3 credentials
uses: aws-actions/configure-aws-credentials@v4
with:
host: interop.seemann.io
username: ${{ secrets.INTEROP_SEEMANN_IO_USER }}
key: ${{ secrets.INTEROP_SEEMANN_IO_SSH_KEY }}
envs: LOGNAME
script: |
cd /mnt/logs
jq '. += [ "${{ needs.config.outputs.logname }}" ]' logs.json | sponge logs.json
rm latest || true
ln -s $LOGNAME latest
aws-access-key-id: ${{ secrets.BACKBLAZE_KEY_ID }}
aws-secret-access-key: ${{ secrets.BACKBLAZE_APPLICATION_KEY }}
- run: aws s3 ls --endpoint-url=https://s3.us-west-000.backblazeb2.com
75 changes: 0 additions & 75 deletions implementations.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,84 +4,9 @@
"url": "https://github.com/lucas-clemente/quic-go",
"role": "both"
},
"ngtcp2": {
"image": "ghcr.io/ngtcp2/ngtcp2-interop:latest",
"url": "https://github.com/ngtcp2/ngtcp2",
"role": "both"
},
"quant": {
"image": "ntap/quant:interop",
"url": "https://github.com/NTAP/quant",
"role": "both"
},
"mvfst": {
"image": "lnicco/mvfst-qns:latest",
"url": "https://github.com/facebookincubator/mvfst",
"role": "both"
},
"quiche": {
"image": "cloudflare/quiche-qns:latest",
"url": "https://github.com/cloudflare/quiche",
"role": "both"
},
"kwik": {
"image": "peterdoornbosch/kwik_n_flupke-interop",
"url": "https://github.com/ptrd/kwik",
"role": "both"
},
"picoquic": {
"image": "privateoctopus/picoquic:latest",
"url": "https://github.com/private-octopus/picoquic",
"role": "both"
},
"aioquic": {
"image": "aiortc/aioquic-qns:latest",
"url": "https://github.com/aiortc/aioquic",
"role": "both"
},
"neqo": {
"image": "neqoquic/neqo-qns:latest",
"url": "https://github.com/mozilla/neqo",
"role": "both"
},
"nginx": {
"image": "public.ecr.aws/nginx/nginx-quic-qns:latest",
"url": "https://quic.nginx.org/",
"role": "server"
},
"msquic": {
"image": "ghcr.io/microsoft/msquic/qns:main",
"url": "https://github.com/microsoft/msquic",
"role": "both"
},
"chrome": {
"image": "martenseemann/chrome-quic-interop-runner",
"url": "https://github.com/marten-seemann/chrome-quic-interop-runner",
"role": "client"
},
"xquic": {
"image": "ghcr.io/alibaba/xquic/xquic-interop:latest",
"url": "https://github.com/alibaba/xquic",
"role": "both"
},
"lsquic": {
"image": "litespeedtech/lsquic-qir:latest",
"url": "https://github.com/litespeedtech/lsquic",
"role": "both"
},
"haproxy": {
"image": "haproxytech/haproxy-qns:latest",
"url": "https://github.com/haproxy/haproxy",
"role": "server"
},
"quinn": {
"image": "stammw/quinn-interop:latest",
"url": "https://github.com/quinn-rs/quinn",
"role": "both"
},
"s2n-quic": {
"image": "public.ecr.aws/s2n/s2n-quic-qns:latest",
"url": "https://github.com/aws/s2n-quic",
"role": "both"
}
}

0 comments on commit 1c0bc2d

Please sign in to comment.