Merge pull request #1 from FedML-AI/master

Update the repo

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 .idea
 .idea/*
 
@@ -27,7 +28,24 @@ wandb
 cifar-10-batches-py
 *.tar.gz
 *.png
+*.zip
+*.json
 
 data/cifar10/*-python
 data/cifar100/*-python
-data/cinic10/
+data/cinic10/
+
+data/MNIST/train
+data/MNIST/test
+data/shakespeare/train
+data/shakespeare/test
+
+
+# android
+.gradle
+fedml_mobile/android/.idea
+fedml_mobile/android/app/build
+fedml_mobile/android/fedml-iot-sdk/build
+fedml_mobile/android/build
+
+mimiconda.sh

.travis.yml

@@ -0,0 +1,16 @@
+env:
+    - ARCH=x86
+dist: bionic
+os:
+    - linux
+language: python
+sudo: false
+python:
+    - "3.7.4"
+install:
+    - "bash CI-install.sh"
+script:
+    - "bash CI-script-framework.sh"
+    - "bash CI-script-fedavg.sh"
+    - "bash CI-script-fednas.sh"
+    - "bash CI-script-fedavg-robust.sh"

CI-install.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+set -ex
+
+# install pyflakes to do code error checking
+echo "pip3 install pyflakes --cache-dir $HOME/.pip-cache"
+pip3 install pyflakes --cache-dir $HOME/.pip-cache
+
+# Conda Installation
+wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+bash miniconda.sh -b -p $HOME/miniconda
+source "$HOME/miniconda/etc/profile.d/conda.sh"
+hash -r
+conda config --set always_yes yes --set changeps1 no
+conda update -q conda
+conda info -a
+
+echo "conda create -n fedml python=3.7.4"
+conda create -n fedml python=3.7.4
+
+echo "conda activate fedml"
+conda activate fedml
+
+# Install PyTorch (please visit pytorch.org to check your version according to your physical machines
+conda install pytorch torchvision cudatoolkit=10.2 -c pytorch
+
+# Install MPI
+conda install -c anaconda mpi4py
+
+# Install Wandb
+pip install --upgrade wandb
+
+# Install other required package
+conda install scikit-learn
+conda install numpy
+conda install setproctitle
+conda install networkx
+
+cd ./fedml_mobile/server/executor
+pip install -r requirements.txt
+cd ./../../../
+
+# install the dataset
+# 1. MNIST
+cd ./data/MNIST
+sh download_and_unzip.sh
+cd ../../
+
+# 2. CIFAR10
+cd ./data/cifar10
+sh download_cifar10.sh
+cd ../../

CI-script-fedavg-robust.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+# code checking
+pyflakes .
+
+# activate the fedml environment
+source "$HOME/miniconda/etc/profile.d/conda.sh"
+conda activate fedml
+
+wandb login ee0b5f53d949c84cee7decbe7a629e63fb2f8408
+wandb off
+
+# 1. CIFAR10 FedAvg-Robust
+cd ./fedml_experiments/distributed/fedavg_robust
+sh run_fedavg_robust_distributed_pytorch.sh 10 10 1 8 resnet56 homo 100 20 64 0.001 cifar10 "./../../../data/cifar10" weak-dp 5.0 0.025 &
+
+sleep 120
+killall mpirun
+cd ./../../../

CI-script-fedavg.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -ex
+
+# code checking
+pyflakes .
+
+# activate the fedml environment
+source "$HOME/miniconda/etc/profile.d/conda.sh"
+conda activate fedml
+
+wandb login ee0b5f53d949c84cee7decbe7a629e63fb2f8408
+wandb off
+
+# 1. MNIST standalone FedAvg
+cd ./fedml_experiments/standalone/fedavg
+sh run_fedavg_standalone_pytorch.sh 2 10 10 mnist ./../../../data/mnist lr hetero 2 2 0.03
+cd ./../../../
+
+
+
+# 2. MNIST distributed FedAvg
+cd ./fedml_experiments/distributed/fedavg
+sh run_fedavg_distributed_pytorch.sh 4 4 1 4 lr hetero 2 2 2 0.03 mnist "./../../../data/mnist" &
+
+sleep 60
+killall mpirun
+cd ./../../../
+
+# 3. MNIST mobile FedAvg
+cd ./fedml_mobile/server/executor/
+python3 app.py &
+bg_pid_server=$!
+echo "pid="$bg_pid_server
+
+sleep 30
+python3 ./mobile_client_simulator.py --client_uuid '0' &
+bg_pid_client0=$!
+echo $bg_pid_client0
+
+python3 ./mobile_client_simulator.py --client_uuid '1' &
+bg_pid_client1=$!
+echo $bg_pid_client1
+
+sleep 80
+kill $bg_pid_server
+kill $bg_pid_client0
+kill $bg_pid_client1
+
+cd ./../../../

CI-script-fednas.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+# code checking
+pyflakes .
+
+# activate the fedml environment
+source "$HOME/miniconda/etc/profile.d/conda.sh"
+conda activate fedml
+
+wandb login ee0b5f53d949c84cee7decbe7a629e63fb2f8408
+wandb off
+
+# 1. search
+cd ./fedml_experiments/distributed/fednas/
+sh run_fednas_search.sh 1 4 darts homo 50 5 2 &
+
+sleep 60
+killall mpirun
+cd ./../../../
+
+# 2. train
+cd ./fedml_experiments/distributed/fednas/
+sh run_fednas_train.sh 1 4 darts homo 500 15 2 &
+
+sleep 60
+killall mpirun
+cd ./../../../

CI-script-framework.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+# code checking
+pyflakes .
+
+# activate the fedml environment
+source "$HOME/miniconda/etc/profile.d/conda.sh"
+conda activate fedml
+
+wandb login ee0b5f53d949c84cee7decbe7a629e63fb2f8408
+wandb off
+
+# 1. distributed base framework
+cd ./fedml_experiments/distributed/base
+sh run_base_distributed_pytorch.sh &
+
+sleep 15
+killall mpirun
+cd ./../../../
+
+# 2. decentralized base framework
+cd ./fedml_experiments/distributed/decentralized_demo
+sh run_decentralized_demo_distributed_pytorch.sh &
+
+sleep 15
+killall mpirun
+cd ./../../../

INSTALL.md

@@ -0,0 +1,5 @@
+## Installation
+
+Please follow CI-install.sh for step-by-step installation.
+
+