Merge remote-tracking branch 'upstream/main'

arayabrain · Jun 21, 2023 · 2f9839c · 2f9839c
2 parents 9cfe433 + 4b5ef78
commit 2f9839c
Show file tree

Hide file tree

Showing 1,042 changed files with 1,747,967 additions and 2,620 deletions.
diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml
@@ -0,0 +1,39 @@
+name: Python Linters
+on:
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - 'notebooks/**'
+      - 'frontend/**'
+      - 'sample_data/**'
+
+jobs:
+  flake8:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - run: python -m pip install flake8
+      - uses: liskin/gh-problem-matcher-wrap@d8afa2cfb66dd3f982b1950429e652bc14d0d7d2
+        with:
+          linters: flake8
+          run: flake8
+  isort:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - run: python -m pip install isort
+      - uses: liskin/gh-problem-matcher-wrap@d8afa2cfb66dd3f982b1950429e652bc14d0d7d2
+        with:
+          linters: isort
+          run: isort --check --diff optinist
+  black:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: psf/black@stable
diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@ __pycache__
 .ipynb_checkpoints
 .vscode/*
 !.vscode/settings.json
+!.vscode/extensions.json
 dist
 /build
 optinist/frontend

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -0,0 +1,14 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+sphinx:
+  configuration: docs/conf.py
+python:
+  install:
+  - requirements: docs/requirements.txt
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -0,0 +1,7 @@
+{
+  "recommendations": [
+    "ms-python.black-formatter",
+    "ms-python.flake8",
+    "ms-python.isort"
+  ]
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -3,5 +3,16 @@
     "frontend/build/**": true,
     "build/**": true
   },
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter",
+    "editor.formatOnSave": true,
+    "editor.codeActionsOnSave": {
+      "source.organizeImports": true
+    }
+  },
+  "python.linting.enabled": true,
+  "python.linting.lintOnSave": true,
+  "python.linting.pylintEnabled": false,
+  "python.linting.flake8Enabled": true,
   "esbonio.sphinx.confDir": ""
 }
diff --git a/Dockerfile b/Dockerfile
@@ -1,23 +1,12 @@
-FROM python:3.9.7-slim
+FROM --platform=linux/amd64 python:3.8.16-slim
 
 COPY requirements.txt /app/requirements.txt
 
 RUN apt-get --allow-releaseinfo-change update && \
-    apt-get install --no-install-recommends -y git gcc g++ libgl1 libgl1-mesa-dev && \
+    apt-get install --no-install-recommends -y gcc g++ libgl1 libgl1-mesa-dev && \
     pip3 install --no-cache-dir --upgrade pip && \
     pip3 install --no-cache-dir -r /app/requirements.txt && \
-    pip3 install --no-cache-dir \
-        cython==0.29.30 \
-        holoviews==1.14.9 \
-        ipyparallel==8.4.1 \
-        matplotlib==3.5.2 \
-        opencv-python==4.6.0.66 \
-        scikit-image==0.18.0 \
-        scikit-learn==1.1.1 \
-        tensorflow==2.9.1 && \
-        watershed==2.2.2 \
-    pip3 install --no-cache-dir git+https://github.com/flatironinstitute/CaImAn.git@914324989443fac5d481ef32aad4f327701294a8#egg=caiman && \
-    apt-get purge git -y && apt-get autoremove -y && apt-get clean && rm -rf /root/.cache/pip/*
+    apt-get clean && rm -rf /root/.cache/pip/*
 
 RUN mkdir -p /root/miniconda3 && \
     apt-get install --no-install-recommends -y wget && \

diff --git a/Dockerfile-test b/Dockerfile-test
@@ -1,4 +1,4 @@
-FROM python:3.9.7-slim
+FROM --platform=linux/amd64 python:3.8.16-slim
 
 COPY requirements.txt /app/requirements.txt
 

diff --git a/Makefile b/Makefile
@@ -46,6 +46,8 @@ local_build:
 
 .PHONY: upload_testpypi
 upload_testpypi:
+	mkdir -p optinist/frontend/build
+	cp -r frontend/build optinist/frontend/
 	python setup.py sdist
 	twine upload --repository testpypi dist/*
 
@@ -55,4 +57,9 @@ test_pypi:
 
 .PHONY: push_pypi
 push_pypi:
-	twine upload --repository pypi dist/*
+	twine upload --repository pypi dist/*
+
+.PHONY: test_doc
+test_doc:
+	python3 -m pip install -r docs/requirements.txt
+	sphinx-autobuild -b html docs docs/_build --port 8001
diff --git a/README.md b/README.md
@@ -37,6 +37,7 @@ OptiNiSt also supports reproducibility of scientific research, standardization o
 ### ROI detection
 - [x] [Suite2p](https://github.com/MouseLand/suite2p)
 - [x] [CaImAn](https://github.com/flatironinstitute/CaImAn)
+- [x] [LCCD](https://github.com/magnetizedCell/lccd-python)
 
 ### Postprocessing
 - [x] Basic Neural Analysis(Event Trigger Average...)
@@ -119,6 +120,7 @@ Kenji Doya, Yukako Yamane [OIST Neural Computation Unit](https://groups.oist.jp/
 ## References
 [[Suite2p]](https://github.com/MouseLand/suite2p) Marius Pachitariu, Carsen Stringer, Mario Dipoppa, Sylvia Schröder, L. Federico Rossi, Henry Dalgleish, Matteo Carandini, Kenneth D. Harris. "Suite2p: beyond 10,000 neurons with standard two-photon microscopy". 2017  
 [[CaImAn]](https://github.com/flatironinstitute/CaImAn) Andrea Giovannucci Is a corresponding author, Johannes Friedrich, Pat Gunn, Jérémie Kalfon, Brandon L Brown, Sue Ann Koay, Jiannis Taxidis, Farzaneh Najafi, Jeffrey L Gauthier, Pengcheng Zhou, Baljit S Khakh, David W Tank, Dmitri B Chklovskii, Eftychios A Pnevmatikakis. "CaImAn: An open source tool for scalable Calcium Imaging data Analysis". 2019  
+[[LCCD]](https://github.com/magnetizedCell/lccd-python) Tsubasa Ito, Keisuke Ota, Kanako Ueno, Yasuhiro Oisi, Chie Matsubara, Kenta Kobayashi, Masamichi Ohkura, Junichi Nakai, Masanori Murayama, Toru Aonishi, "Low computational-cost cell detection method for calcium imaging data", 2022
 [[PyNWB]](https://github.com/NeurodataWithoutBorders/pynwb) Oliver Rübel, Andrew Tritt, Ryan Ly, Benjamin K. Dichter, Satrajit Ghosh, Lawrence Niu, Ivan Soltesz, Karel Svoboda, Loren Frank, Kristofer E. Bouchard, "The Neurodata Without Borders ecosystem for neurophysiological data science", bioRxiv 2021.03.13.435173, March 15, 2021
 
 <!-- ## Citing the Project

diff --git a/docs/Makefile b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/_static/installation/snakemake_use_conda.png b/docs/_static/installation/snakemake_use_conda.png
diff --git a/docs/_static/tutorials/caiman_sample_pipeline.png b/docs/_static/tutorials/caiman_sample_pipeline.png
diff --git a/docs/_static/tutorials/edit-roi/add_roi_clicked.png b/docs/_static/tutorials/edit-roi/add_roi_clicked.png
diff --git a/docs/_static/tutorials/edit-roi/box.png b/docs/_static/tutorials/edit-roi/box.png
diff --git a/docs/_static/tutorials/edit-roi/cell_roi_selected.png b/docs/_static/tutorials/edit-roi/cell_roi_selected.png
diff --git a/docs/_static/tutorials/edit-roi/roi_selected_merge_or_delete.png b/docs/_static/tutorials/edit-roi/roi_selected_merge_or_delete.png
diff --git a/docs/_static/tutorials/fig11_runall.png b/docs/_static/tutorials/fig11_runall.png
diff --git a/docs/_static/tutorials/fig12_cancel.png b/docs/_static/tutorials/fig12_cancel.png
diff --git a/docs/_static/tutorials/fig2.2_tags.png b/docs/_static/tutorials/fig2.2_tags.png
diff --git a/docs/_static/tutorials/fig2_open.png b/docs/_static/tutorials/fig2_open.png
diff --git a/docs/_static/tutorials/fig3_imagenode.png b/docs/_static/tutorials/fig3_imagenode.png
diff --git a/docs/_static/tutorials/fig5_imagenode2.png b/docs/_static/tutorials/fig5_imagenode2.png
diff --git a/docs/_static/tutorials/fig6_suite2p.png b/docs/_static/tutorials/fig6_suite2p.png
diff --git a/docs/_static/tutorials/fig7_suite2p2.png b/docs/_static/tutorials/fig7_suite2p2.png
diff --git a/docs/_static/tutorials/fig8_suite2pcaiman.png b/docs/_static/tutorials/fig8_suite2pcaiman.png
diff --git a/docs/_static/tutorials/fig9_workflowbranch.png b/docs/_static/tutorials/fig9_workflowbranch.png
diff --git a/docs/_static/tutorials/lccd_sample_pipeline.png b/docs/_static/tutorials/lccd_sample_pipeline.png
diff --git a/docs/_static/tutorials/sample_pipelines.png b/docs/_static/tutorials/sample_pipelines.png
diff --git a/docs/_static/tutorials/suite2p_sample_pipeline.png b/docs/_static/tutorials/suite2p_sample_pipeline.png
diff --git a/docs/conf.py b/docs/conf.py
@@ -7,8 +7,9 @@
 import os
 import sys
 from datetime import datetime
-sys.path.insert(0, os.path.abspath('../'))
-from optinist.version import VERSION, VERSION_SHORT
+
+sys.path.insert(0, os.path.abspath("../"))
+from optinist.version import VERSION, VERSION_SHORT  # noqa: E402
 
 # -- Path setup --------------------------------------------------------------
 
@@ -20,17 +21,17 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'OptiNiSt'
+project = "OptiNiSt"
 copyright = f"{datetime.today().year}, OIST"
-author = ''
+author = ""
 version = VERSION_SHORT
 release = VERSION
 
 # -- readthedocs -------------------------------------------------------------
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+on_rtd = os.environ.get("READTHEDOCS", None) == "True"
 
 # -- General configuration ---------------------------------------------------
-master_doc = 'index'
+master_doc = "index"
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -40,15 +41,15 @@
     "sphinx.ext.intersphinx",
     "sphinx.ext.mathjax",
     "sphinx.ext.viewcode",
-    'sphinx.ext.napoleon',
-    'sphinx.ext.coverage',
-    'sphinxcontrib.apidoc',
-    'sphinx_autodoc_typehints',
-    'myst_parser',
-    'sphinx.ext.autosectionlabel',
-    'sphinx.ext.autosummary',
-    'sphinx.ext.extlinks',
-    'sphinx.ext.autodoc.typehints'
+    "sphinx.ext.napoleon",
+    "sphinx.ext.coverage",
+    "sphinxcontrib.apidoc",
+    "sphinx_autodoc_typehints",
+    "myst_parser",
+    "sphinx.ext.autosectionlabel",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.extlinks",
+    "sphinx.ext.autodoc.typehints",
 ]
 
 # Tell myst-parser to assign header anchors for h1-h3.
@@ -57,12 +58,12 @@
 suppress_warnings = ["myst.header"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'tests', '*test*']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "tests", "*test*"]
 
 source_suffix = [".rst", ".md"]
 
@@ -71,31 +72,31 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-html_logo = '_static/optinist.png'
-html_favicon = '_static/favicon.ico'
+html_static_path = ["_static"]
+html_logo = "_static/optinist.png"
+html_favicon = "_static/favicon.ico"
 
 # disable document page source link
 html_show_sourcelink = False
 
 autosummary_generate = True
 
 html_theme_options = {
-    'canonical_url': '',
-    'logo_only': False,
-    'display_version': True,
-    'prev_next_buttons_location': 'top',
-    'style_external_links': False,
-    'style_nav_header_background': '#C3EBE1',
+    "canonical_url": "",
+    "logo_only": False,
+    "display_version": True,
+    "prev_next_buttons_location": "top",
+    "style_external_links": False,
+    "style_nav_header_background": "#C3EBE1",
     # Toc options
-    'collapse_navigation': True,
-    'sticky_navigation': True,
-    'navigation_depth': 4,
-    'includehidden': True,
-    'titles_only': False,
-}
+    "collapse_navigation": True,
+    "sticky_navigation": True,
+    "navigation_depth": 4,
+    "includehidden": True,
+    "titles_only": False,
+}
diff --git a/docs/gui/record.md b/docs/gui/record.md
@@ -1,21 +1,28 @@
 Record
 =================
-In the RECORD field, you can check the workflow status in your OPTINIST_DIR and manage your analysis pipeline. The table lists all the pipelines in your OPTINIST_DIR. By clicking on the mark on the 2nd column, you can show the details of each pipeline.
-
-OptiNiSt records and reproduces past workflow pipelines. It can download results in nwb format.
 <br>
 <p align="center">
 <img width="400px" src="../_static/record/whole.png" alt="Whole"/>
 </p>
 
 
+In the RECORD field, you can check the workflow status in your OPTINIST_DIR and manage your analysis pipeline. The table lists all the pipelines in your OPTINIST_DIR. By clicking on the mark on the 2nd column, you can show the details of each pipeline.
+
+OptiNiSt records and reproduces past workflow pipelines. It can download results in nwb format.
+
+
 OptiNiSt can:
 - Record past executed workflow
 - Reproduce past workflow
 - Download results as NWB files
 - Download snakemake config file
 
 ### Record Table
+
+<p align="center">
+<img width="400px" src="../_static/record/components/table.png" alt="Table"/>
+</p>
+
 - **Timestamp**: latest execution date and time.
 - **ID**: unique ID. This is the directory name for the whole results of the pipeline.
 - **Name**: user-defined workflow name.
@@ -25,18 +32,39 @@ OptiNiSt can:
 - **NWB**: button to copy the analysis results as NWB file to your download folder on your computer.
 - **Delete**: button to delete the workflow from the OPTINIST_DIR.
 
-<p align="center">
-<img width="400px" src="../_static/record/components/table.png" alt="Table"/>
-</p>
 
 
 ### Details
+
+<p align="center">
+<img width="400px" src="../_static/record/components/details.png" alt="Details"/>
+</p>
+
 - **Function**: names of the nods (function).
 - **nodeID**: unique ID. This is the directory name for the results of each node.
 - **Success**: success or failure (abortion with error) of execution of the node.
 - **NWB**: button to copy the analysis results as NWB file to your download folder on your computer.
 
 
-<p align="center">
-<img width="400px" src="../_static/record/components/details.png" alt="Details"/>
+### Reproduce Button
+
+<br>
+<p align="left">
+<img width="100px" src="../_static/tutorials/fig41_reproduce.png" alt="Whole" />
 </p>
+
+Clicking the Reproduce arrow retrieves the pipeline onto the workflow. This function is convenient when you restart the analysis after closing the browser. The reproduced pipeline needs to be `RUN` again (not `RUN ALL`) to make plots available.
+
+### Download Buttons
+
+<br>
+<p align="left">
+<img width="150px" src="../_static/tutorials/fig42_workflownwb.png" alt="Whole" />
+</p>
+
+The Download buttons for the Workflow column and the NWB column copy the Snakemake config or NWB file to your download folder. 
+
+- The Snakemake config file contains the workflow information and parameters for each node. 
+- The NWB file contains the data and its analysis results. 
+
+This function is convenient when users want to share the same analysis pipeline or inspect the output results.