Merge pull request #1 from alan-turing-institute/pytorch_GAN_zoo

Pytorch gan zoo

.github/workflows/build.yaml

@@ -0,0 +1,44 @@
+---
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Install singularity
+        run: |
+          sudo apt-get update
+          # Install singularity dependencies
+          sudo apt-get install -y \
+          build-essential \
+          libssl-dev \
+          uuid-dev \
+          libgpgme11-dev \
+          squashfs-tools \
+          libseccomp-dev \
+          wget \
+          pkg-config \
+          git \
+          cryptsetup \
+          golang
+          # Get singularity release
+          export VERSION=3.8.0
+          wget https://github.com/hpcng/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz
+          tar -xzf singularity-${VERSION}.tar.gz
+          # Build/install singularity
+          cd singularity-${VERSION}
+          ./mconfig
+          make -C builddir
+          sudo make -C builddir install
+
+      - name: Build pytorch GAN zoo
+        working-directory: workflows/pytorch_GAN_zoo/
+        run: ./build.sh

.github/workflows/lint.yaml

@@ -0,0 +1,46 @@
+---
+name: Lint
+
+on: [push, pull_request]
+
+jobs:
+  markdown_lint:
+    name: "Lint Markdown"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Ruby
+        uses: actions/setup-ruby@v1
+
+      - name: Install mdl
+        run: gem install mdl
+
+      - name: Lint markdown
+        run: mdl --style .mdl_style.rb ./
+
+  markdown_link_check:
+    name: "Check Markdown links"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Check links
+        uses: gaurav-nelson/github-action-markdown-link-check@1.0.13
+        with:
+          use-verbose-mode: "yes"
+
+  shellcheck:
+    name: "Check shell scripts"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: ShellCheck
+        uses: ludeeus/action-shellcheck@1.1.0

.gitignore

@@ -0,0 +1,5 @@
+# Singularity image files
+*.sif
+
+# Vagrant directory
+.vagrant

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "pytorch_GAN_zoo"]
+	path = workflows/pytorch_GAN_zoo/pytorch_GAN_zoo
+	url = https://github.com/facebookresearch/pytorch_GAN_zoo.git

.mdl_style.rb

@@ -0,0 +1,20 @@
+# mdl rules https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md
+
+# Import all default rules
+all
+
+# Only allow atx style headings (e.g. # H1 ## H2)
+rule 'MD003', :style => :atx
+
+# Only allow dashes in unordered lists
+rule 'MD004', :style => :dash
+
+# Enforce line length of 80 characters except in code blocks and tables
+rule 'MD013', :code_blocks => false, :tables => false
+
+# Ignore blockquotes separated only be a blank line. This is a limitation of
+# some markdown parsers, not markdown itself.
+exclude_rule 'MD028'
+
+# Allow bare URLs (i.e. without angle brackets)
+exclude_rule 'MD034'

README.md

@@ -1,2 +1,3 @@
 # AI-workflows
+
 A collections of portable, real-world AI workflows for testing and benchmarking

Vagrantfile

@@ -0,0 +1,41 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/ubuntu2004"
+
+  config.vm.hostname = "singularity"
+
+  config.vm.synced_folder "./", "/vagrant"
+
+   config.vm.provider "virtualbox" do |vb|
+     vb.memory = "8192"
+     vb.cpus = "4"
+   end
+
+  config.vm.provision "shell", inline: <<-SHELL
+    # Install singularity dependencies
+    apt-get update
+    apt-get install -y \
+    build-essential \
+    libssl-dev \
+    uuid-dev \
+    libgpgme11-dev \
+    squashfs-tools \
+    libseccomp-dev \
+    wget \
+    pkg-config \
+    git \
+    cryptsetup \
+    golang
+    apt-get clean
+    # Get singularity release
+    export VERSION=3.8.0
+    wget https://github.com/hpcng/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz
+    tar -xzf singularity-${VERSION}.tar.gz
+    # Build/install singularity
+    cd singularity-${VERSION}
+    ./mconfig
+    make -C builddir
+    make -C builddir install
+    cd ../
+    rm -r singularity-${VERSION} singularity-${VERSION}.tar.gz
+  SHELL
+end

workflows/pytorch_GAN_zoo/README.md

@@ -0,0 +1,48 @@
+# PyTorch GAN Zoo
+
+This example builds a singularity container for [Facebook Research's PyTorch GAN
+Zoo](https://github.com/facebookresearch/pytorch_GAN_zoo).
+
+The singularity container will allow you to call all the scripts from the
+project and includes are requirements. The container supports CUDA versions
+10.1, 10.2 and 11.1 on the host.
+
+## Building
+
+To build the singularity container use the build script in this directory.
+
+```bash
+./build.sh
+```
+
+This script will try to use singularities fakeroot support if you run as a
+non-root user. If this is not supported on your system you can run the script as
+root.
+
+When the script is finished you will find the container (`pytorch_GAN_zoo.sif`)
+in you current working directory.
+
+## Usage
+
+The scripts from [PyTorch GAN
+Zoo](https://github.com/facebookresearch/pytorch_GAN_zoo) can be called with
+`singularity exec pytorch_GAN_zoo.sif <script name>`, for example
+
+```bash
+singularity exec pytorch_GAN_zoo.sif eval.py
+```
+
+Any flags or command line arguments can be declared after the script name.
+
+When training, you will need to supply the `--nv` flag to singularity so that
+the host GPU may be used. You will also need to select a singularity app, using
+the `--app` flag to select the appropriate CUDA version. The available apps are
+`cu101`, `cu102`, and `cu111` for CUDA 10.1, 10.2 and 11.1 respectively.
+
+For example, to pre-process the celeba dataset and train a PGAN model on a host
+with CUDA 10.2 you could run the following commands.
+
+```bash
+singularity exec pytorch_GAN_zoo.sif datasets.py celeba_cropped <path to celeba dataset>/img_align_celeba/ -o celeba
+singularity exec pytorch_GAN_zoo.sif --nv --app cu102 train.py PGAN -c config_celeba_cropped.json --restart -n celeba_cropped
+```

workflows/pytorch_GAN_zoo/build.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+UID=$(id -u)
+DEF_FILE="pytorch_GAN_zoo.def"
+SIF_FILE="pytorch_GAN_zoo.sif"
+
+if [ "$UID" = 0 ]; then
+    singularity build $SIF_FILE $DEF_FILE
+else
+    singularity build --fakeroot $SIF_FILE $DEF_FILE
+fi

workflows/pytorch_GAN_zoo/pytorch_GAN_zoo

@@ -0,0 +1 @@
+Subproject commit b75dee40918caabb4fe7ec561522717bf096a8cb

workflows/pytorch_GAN_zoo/pytorch_GAN_zoo.def

@@ -0,0 +1,75 @@
+BootStrap: library
+From: ubuntu:20.04
+
+# Global settings
+%files
+    pytorch_GAN_zoo/ /
+
+%environment
+    # Add pytorch GAN zoo directory to path
+    export PATH=$PATH:/pytorch_GAN_zoo
+
+%post
+    apt-get -y update
+
+    # Add universe repository (necessary for python3-pip)
+    apt-get -y install software-properties-common
+    add-apt-repository -y -u universe
+
+    # Install python packages
+    apt-get -y install python3 python3-pip python3-venv
+
+    # Make Python scripts executable
+    cd /pytorch_GAN_zoo
+    pwd
+    ls
+    for i in *.py; do sed -i "1s|^|#!/usr/bin/env python3\n|" $i; done
+    chmod a+x *.py
+
+    # Install python dependencies
+    pip3 install --no-cache-dir -r requirements.txt
+    pip3 install --no-cache-dir imageio  # For image processing
+
+
+# CUDA 11.1 app
+%apphelp cu111
+    Cuda 11.1 support, torch 1.9.0, torchvision 0.10.0, torchaudio 0.9.0
+
+%appinstall cu111
+    python3 -m venv --system-site-packages ./venv_cu111
+    . ./venv_cu111/bin/activate
+    pip3 install --no-cache-dir torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
+    deactivate
+
+%appenv cu111
+    PATH="/scif/apps/cu111/venv_cu111/bin:$PATH"
+    export PATH
+
+# CUDA 10.2 app
+%apphelp cu102
+    Cuda 10.2 support, torch 1.9.0, torchvision 0.10.0, torchaudio 0.9.0
+
+%appinstall cu102
+    python3 -m venv --system-site-packages ./venv_cu102
+    . ./venv_cu102/bin/activate
+    pip3 install --no-cache-dir torch torchvision torchaudio
+    deactivate
+
+%appenv cu102
+    PATH="/scif/apps/cu102/venv_cu102/bin:$PATH"
+    export PATH
+
+
+# CUDA 10.1 app
+%apphelp cu101
+    Cuda 10.1 support, torch 1.7.1, torchvision 0.8.2, torchaudio 0.7.2
+
+%appinstall cu101
+    python3 -m venv --system-site-packages ./venv_cu101
+    . ./venv_cu101/bin/activate
+    pip3 install --no-cache-dir torch==1.7.1+cu101 torchvision==0.8.2+cu101 torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html
+    deactivate
+
+%appenv cu101
+    PATH="/scif/apps/cu101/venv_cu101/bin:$PATH"
+    export PATH