Skip to content

Added script relevant for the pseudo-bulk expression and UMAP coordinates rendering #327

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
StevenQiaoUT wants to merge 2 commits into
base: dev
Choose a base branch
from
Open

Added script relevant for the pseudo-bulk expression and UMAP coordinates rendering #327

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1a9617a
standardized procedure for db
StevenQiaoUT Jun 30, 2026
498005f
Added all the correct pseudobulk and umap information
StevenQiaoUT Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions api/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ def create_app():
from api.resources.llama3 import llama3
from api.resources.gene_expression import gene_expression
from api.resources.gene_density import gene_density
from api.resources.umap_expression import umap_expression

bar_api.add_namespace(gene_information)
bar_api.add_namespace(gaia)
Expand All @@ -110,6 +111,7 @@ def create_app():
bar_api.add_namespace(llama3)
bar_api.add_namespace(gene_expression)
bar_api.add_namespace(gene_density)
bar_api.add_namespace(umap_expression)
bar_api.init_app(bar_app)
return bar_app

Expand Down
15 changes: 15 additions & 0 deletions api/models/efp_schemas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,21 @@ def _schema(species: str, charset: str = "latin1") -> DatabaseSpec:
("wheat_meiosis", "wheat"),
("wheat_root", "wheat"),
("willow", "willow"),
("arabidopsis_NIE_pseudobulk", "arabidopsis"),
("rice_OW_pseudobulk", "rice"),
("rice_OW_umap", "rice"),
("arabidopsis_stem_lee_pseudobulk", "arabidopsis"),
("arabidopsis_flower_lee_pseudobulk", "arabidopsis"),
("arabidopsis_silique_lee_pseudobulk", "arabidopsis"),
("arabidopsis_root_rs_pseudobulk", "arabidopsis"),
("arabidopsis_seed_martin_pseudobulk", "arabidopsis"),
("arabidopsis_NIE_umap", "arabidopsis"),
("arabidopsis_root_shahan_umap", "arabidopsis"),
("arabidopsis_seed_martin_umap", "arabidopsis"),
("arabidopsis_flower_lee_umap", "arabidopsis"),
("arabidopsis_silique_lee_umap", "arabidopsis"),
("arabidopsis_stem_lee_umap", "arabidopsis"),

]

# Databases that store Affymetrix/microarray probeset IDs instead of gene identifiers.
Expand Down
148 changes: 148 additions & 0 deletions api/resources/umap_expression.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
"""
Steven Qiao | BCB330 Project 2025-2026 | University of Toronto

REST endpoint for per-cell UMAP coordinate + expression queries.

Routes: GET /umap_expression/umap/<database>/<gene_id>

All gene IDs are validated by species before reaching the query layer.
Expression is stored as a JSON array per gene in umap_expression; UMAP
coordinates are stored once per dataset in umap_coords. The two are
merged server-side by position before returning to the client.
"""
import json

from flask_restx import Namespace, Resource
from markupsafe import escape
from sqlalchemy import text
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session

from api import db
from api.utils.bar_utils import BARUtils
from api.utils.gene_id_utils import (
CROSS_SPECIES_DATABASES,
DATABASE_SPECIES,
normalize_gene_id,
validate_gene_id,
)

umap_expression = Namespace(
"UMAP Expression",
description="Per-cell UMAP coordinates and expression data for SUPeR Viewer",
path="/umap_expression",
)

# Maps UMAP database names to their pseudobulk counterpart species.
# Add a new entry here whenever a new UMAP dump is generated.
UMAP_DATABASE_SPECIES: dict[str, str] = {
"rice_OW_umap": "rice",
"arabidopsis_NIE_umap": "arabidopsis",
"arabidopsis_root_shahan_umap": "arabidopsis",
"arabidopsis_seed_martin_umap": "arabidopsis",
"arabidopsis_flower_lee_umap": "arabidopsis",
"arabidopsis_silique_lee_umap": "arabidopsis",
"arabidopsis_stem_lee_umap": "arabidopsis",
}


@umap_expression.route("/<string:database>/<string:gene_id>")
@umap_expression.doc(description="Retrieve per-cell UMAP coordinates and expression values for a gene.")
@umap_expression.param(
"gene_id",
"Gene ID (e.g. AT1G01010 for Arabidopsis, Os10g0168500 for rice)",
_in="path",
default="Os10g0168500",
)
@umap_expression.param(
"database",
"UMAP database name (e.g. rice_OW_umap, arabidopsis_NIE_umap)",
_in="path",
default="rice_OW_umap",
)
class UMAPExpression(Resource):
def get(self, database, gene_id):
"""Retrieve per-cell UMAP coordinates and expression for a gene."""
database = str(escape(database))
gene_id = str(escape(gene_id))

# 1. Resolve database species
species = UMAP_DATABASE_SPECIES.get(database)
if species is None:
return BARUtils.error_exit(
f"Unknown UMAP database '{database}'. "
f"Available: {', '.join(sorted(UMAP_DATABASE_SPECIES.keys()))}"
), 400

# 2. Validate gene ID format against the expected input species regex
input_species = CROSS_SPECIES_DATABASES.get(database, species)
if not validate_gene_id(gene_id, input_species):
return BARUtils.error_exit(f"Invalid {input_species} gene ID: '{gene_id}'"), 400

# 3. Normalise (e.g. strip maize transcript suffix _T##)
gene_id = normalize_gene_id(gene_id, species)

# 4. Get SQLAlchemy bind engine for this database
engine = db.engines.get(database)
if engine is None:
return BARUtils.error_exit("Database not available"), 503

# 5. Query expression JSON array for this gene (single PK lookup)
expr_sql = text(
"SELECT expression FROM umap_expression WHERE gene_id = :gene_id"
)

try:
with Session(engine) as session:
row = session.execute(expr_sql, {"gene_id": gene_id}).first()
except SQLAlchemyError as exc:
return BARUtils.error_exit(f"Database query failed: {str(exc)}"), 500

# Retry with uppercase (some datasets store IDs in uppercase)
if row is None:
try:
with Session(engine) as session:
row = session.execute(expr_sql, {"gene_id": gene_id.upper()}).first()
except SQLAlchemyError as exc:
return BARUtils.error_exit(f"Database query failed: {str(exc)}"), 500

if row is None:
return BARUtils.error_exit("No data found for the given gene"), 404

# Parse expression JSON array: [val0, val1, val2, ...]
expr_raw = row.expression
expr_list = json.loads(expr_raw) if isinstance(expr_raw, str) else expr_raw

# 6. Query all coords ordered by cell_id (same for every gene)
coords_sql = text(
"SELECT cell_id, umap_1, umap_2, cell_type "
"FROM umap_coords ORDER BY cell_id"
)

try:
with Session(engine) as session:
coords = session.execute(coords_sql).all()
except SQLAlchemyError as exc:
return BARUtils.error_exit(f"Database query failed: {str(exc)}"), 500

# 7. Merge coords + expression by position
data = [
{
"umap_1": float(c.umap_1),
"umap_2": float(c.umap_2),
"expression": float(expr_list.get(str(c.cell_id), 0.0)),
"cell_type": str(c.cell_type),
}
for i, c in enumerate(coords)
]

return BARUtils.success_exit({
"gene_id": gene_id,
"database": database,
"species": species,
"record_count": len(data),
"data": data,
})


umap_expression.add_resource(UMAPExpression, "/<string:database>/<string:gene_id>")
16 changes: 10 additions & 6 deletions api/utils/bar_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,15 +69,19 @@ def is_poplar_gene_valid(gene):
def is_rice_gene_valid(gene, isoform_id=False):
"""This function verifies if rice gene is valid
:param gene:
:param isoform_id: True if you want to verifiy isoform ID
:param isoform_id: True if you want to verify isoform ID
:return: True if valid
"""
if isoform_id and re.search(r"^LOC_Os\d{2}g\d{5}\.\d{1,2}$", gene, re.I):
return True
elif isoform_id is False and re.search(r"^LOC_Os\d{2}g\d{5}$", gene, re.I):
return True
if isoform_id:
return bool(
re.search(r"^LOC_Os\d{2}g\d{5}\.\d{1,2}$", gene, re.I) or
re.search(r"^Os\d{2}g\d{7}\.\d{1,2}$", gene, re.I)
)
else:
return False
return bool(
re.search(r"^LOC_Os\d{2}g\d{5}$", gene, re.I) or
re.search(r"^Os\d{2}g\d{7}$", gene, re.I)
)

@staticmethod
def is_tomato_gene_valid(gene, isoform_id=False):
Expand Down
15 changes: 15 additions & 0 deletions api/utils/gene_id_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ def is_probeset_id(gene_id: str) -> bool:
"shoot_apex": "arabidopsis",
"silique": "arabidopsis",
"single_cell": "arabidopsis",

# Actinidia (kiwifruit)
"actinidia_bud_development": "actinidia",
"actinidia_flower_fruit_development": "actinidia",
Expand Down Expand Up @@ -285,6 +286,20 @@ def is_probeset_id(gene_id: str) -> bool:
"willow": "willow",
# Test
"sample_data": "arabidopsis",
'arabidopsis_NIE_pseudobulk': "arabidopsis",
"arabidopsis_stem_lee_pseudobulk": "arabidopsis",
"arabidopsis_flower_lee_pseudobulk": "arabidopsis",
"arabidopsis_silique_lee_pseudobulk": "arabidopsis",
"arabidopsis_root_rs_pseudobulk": "arabidopsis",
"arabidopsis_seed_martin_pseudobulk": "arabidopsis",
"rice_OW_pseudobulk": "rice",
"rice_OW_umap": "rice",
"arabidopsis_NIE_umap": "arabidopsis",
"arabidopsis_root_shahan_umap": "arabidopsis",
"arabidopsis_seed_martin_umap": "arabidopsis",
"arabidopsis_flower_lee_umap": "arabidopsis",
"arabidopsis_silique_lee_umap": "arabidopsis",
"arabidopsis_stem_lee_umap": "arabidopsis",
}
# fmt: on

Expand Down
16 changes: 15 additions & 1 deletion config/BAR_API.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,5 +43,19 @@ SQLALCHEMY_BINDS = {
'tomato_nssnp' : 'mysql://root:root@localhost/tomato_nssnp',
'tomato_sequence' : 'mysql://root:root@localhost/tomato_sequence',
'triphysaria' : 'mysql://root:root@localhost/triphysaria',
'gaia' : 'mysql://root:root@localhost/gaia'
'gaia' : 'mysql://root:root@localhost/gaia',
'rice_OW_pseudobulk': 'mysql://root:root@localhost/rice_OW_pseudobulk',
'arabidopsis_NIE_pseudobulk': 'mysql://root:root@localhost/arabidopsis_NIE_pseudobulk',
'arabidopsis_stem_lee_pseudobulk': 'mysql://root:root@localhost/arabidopsis_stem_lee_pseudobulk',
'arabidopsis_flower_lee_pseudobulk': 'mysql://root:root@localhost/arabidopsis_flower_lee_pseudobulk',
'arabidopsis_silique_lee_pseudobulk': 'mysql://root:root@localhost/arabidopsis_silique_lee_pseudobulk',
'arabidopsis_root_rs_pseudobulk': 'mysql://root:root@localhost/arabidopsis_root_rs_pseudobulk',
'arabidopsis_seed_martin_pseudobulk': 'mysql://root:root@localhost/arabidopsis_seed_martin_pseudobulk',
'rice_OW_umap': 'mysql://root:root@localhost/rice_OW_umap',
'arabidopsis_NIE_umap': 'mysql://root:root@localhost/arabidopsis_NIE_umap',
'arabidopsis_root_shahan_umap': 'mysql://root:root@localhost/arabidopsis_root_shahan_umap',
'arabidopsis_seed_martin_umap': 'mysql://root:root@localhost/arabidopsis_seed_martin_umap',
'arabidopsis_flower_lee_umap': 'mysql://root:root@localhost/arabidopsis_flower_lee_umap',
'arabidopsis_silique_lee_umap': 'mysql://root:root@localhost/arabidopsis_silique_lee_umap',
'arabidopsis_stem_lee_umap': 'mysql://root:root@localhost/arabidopsis_stem_lee_umap'
}
63 changes: 63 additions & 0 deletions config/databases/arabidopsis_NIE_pseudobulk_dump.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
-- MySQL dump 10.13 Distrib 9.4.0, for Linux (x86_64)
--
-- Host: localhost Database: arabidopsis_NIE_pseudobulk
-- ------------------------------------------------------
-- Server version 9.4.0

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!50503 SET NAMES utf8mb4 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Current Database: `arabidopsis_NIE_pseudobulk`
--

CREATE DATABASE /*!32312 IF NOT EXISTS*/ `arabidopsis_NIE_pseudobulk` /*!40100 DEFAULT CHARACTER SET latin1 */ /*!80016 DEFAULT ENCRYPTION='N' */;

USE `arabidopsis_NIE_pseudobulk`;

--
-- Table structure for table `sample_data`
--

DROP TABLE IF EXISTS `sample_data`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `sample_data` (
`data_probeset_id` varchar(16) NOT NULL,
`data_signal` float DEFAULT '0',
`data_signal_std` float DEFAULT '0',
`data_bot_id` varchar(64) NOT NULL,
KEY `data_probeset_id` (`data_probeset_id`,`data_bot_id`,`data_signal`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `sample_data`
--

LOCK TABLES `sample_data` WRITE;
/*!40000 ALTER TABLE `sample_data` DISABLE KEYS */;
INSERT INTO `sample_data` VALUES ('AT1G01010',0.0346533,0.224356,'D0_Mesophyll'),('AT1G01010',0.0251518,0.185286,'D0_Sieve element_responsive'),('AT1G01010',0.0297745,0.203837,'D0_Guard'),('AT1G01010',0.0550332,0.260401,'D0_Defense state'),('AT1G01010',0.0424581,0.239094,'D0_Epidermal'),('AT1G01010',0.0163655,0.146177,'D0_Phloem Parenchyma'),('AT1G01010',0.0389615,0.234467,'D0_Metabolic stress state'),('AT1G01010',0.0292404,0.198967,'D0_Phloem companion'),('AT1G01010',0.0510094,0.226118,'D0_Trichome'),('AT1G01010',0.027103,0.181704,'D0_Dividing'),('AT1G01010',0.0692417,0.276578,'D0_Stress responsive'),('AT1G01010',0.0110158,0.119154,'D0_Sugar metabolic state'),('AT1G01010',0.0565404,0.272046,'D0_Immune active'),('AT1G01010',0.0513775,0.273271,'D0_Hydathode'),('AT1G01010',0.045102,0.247266,'D0_Vascular'),('AT1G01010',0.0256331,0.169289,'D0_Myrosin'),('AT1G01010',0.0274436,0.177235,'W0_Vascular'),('AT1G01010',0.024174,0.172252,'W0_Mesophyll'),('AT1G01010',0.0213485,0.15354,'W0_Phloem Parenchyma'),('AT1G01010',0.0286998,0.178188,'W0_Dividing'),('AT1G01010',0.0278313,0.182473,'W0_Epidermal'),('AT1G01010',0.0563447,0.251333,'W0_Immune active'),('AT1G01010',0.0343054,0.201204,'W0_Sieve element_responsive'),('AT1G01010',0.0574457,0.253535,'W0_Defense state'),('AT1G01010',0.0136855,0.12117,'W0_Guard'),('AT1G01010',0.0272221,0.169833,'W0_Phloem companion'),('AT1G01010',0.0715313,0.271057,'W0_Stress responsive'),('AT1G01010',0.00820466,0.0673068,'W0_Myrosin'),('AT1G01010',0.0393423,0.199799,'W0_Sugar metabolic state'),('AT1G01010',0.0339956,0.217678,'W0_Metabolic stress state'),('AT1G01010',0.0214199,0.133693,'W0_Trichome'),('AT1G01010',0.0571251,0.236552,'W0_Hydathode'),('AT1G01010',0.0422542,0.250415,'W15_Dividing'),('AT1G01010',0.0446,0.254963,'W15_Mesophyll'),('AT1G01010',0.037824,0.236808,'W15_Phloem Parenchyma'),('AT1G01010',0.0420744,0.251218,'W15_Immune active'),('AT1G01010',0.0387755,0.241218,'W15_Epidermal'),('AT1G01010',0.104241,0.43291,'W15_Hydathode'),('AT1G01010',0.104467,0.324257,'W15_Stress responsive'),('AT1G01010',0.03282,0.228891,'W15_Guard'),('AT1G01010',0.0783275,0.325689,'W15_Defense state'),('AT1G01010',0.0205945,0.15169,'W15_Myrosin'),('AT1G01010',0.0290853,0.203305,'W15_Phloem companion'),('AT1G01010',0.0409241,0.246531,'W15_Vascular'),('AT1G01010',0,0,'W15_Trichome'),('AT1G01010',0.0304135,0.206024,'W15_Sieve element_responsive'),('AT1G01010',0,0,'W15_Sugar metabolic state'),('AT1G01010',0.0573618,0.262866,'W15_Metabolic stress state'),('AT1G01010',0.0187317,0.156752,'R15_Sieve element_responsive'),('AT1G01010',0.0287312,0.191696,'R15_Immune active'),('AT1G01010',0.0329913,0.213788,'R15_Mesophyll'),('AT1G01010',0.0130775,0.14382,'R15_Guard'),('AT1G01010',0.0547408,0.257222,'R15_Defense state'),('AT1G01010',0.0541722,0.243631,'R15_Stress responsive'),('AT1G01010',0.02531,0.180358,'R15_Phloem Parenchyma'),('AT1G01010',0.0400045,0.239473,'R15_Epidermal'),('AT1G01010',0.0300884,0.196556,'R15_Vascular'),('AT1G01010',0.0282214,0.199286,'R15_Dividing'),('AT1G01010',0.034919,0.214859,'R15_Phloem companion'),('AT1G01010',0.018792,0.157063,'R15_Metabolic stress state'),('AT1G01010',0.0528136,0.261553,'R15_Trichome'),('AT1G01010',0,0,'R15_Myrosin'),('AT1G01010',0.0564275,0.281679,'R15_Hydathode'),('AT1G01010',0.0462055,0.222434,'R15_Sugar metabolic state');
INSERT INTO `sample_data` VALUES ('AT1G01010',0.0242853,0.114475,'W0_Phloem average');
INSERT INTO `sample_data` VALUES ('AT1G01010',0.0228029,0.123446,'D0_Phloem average');
INSERT INTO `sample_data` VALUES ('AT1G01010',0.0301145,0.140262,'R15_Phloem average');
INSERT INTO `sample_data` VALUES ('AT1G01010',0.0334546,0.156053,'W15_Phloem average');
/*!40000 ALTER TABLE `sample_data` ENABLE KEYS */;
UNLOCK TABLES;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2026-07-17 01:32:31
Loading