diff --git a/web/server/codechecker_server/api/mass_store_run.py b/web/server/codechecker_server/api/mass_store_run.py index a04ef63834..d0396d4516 100644 --- a/web/server/codechecker_server/api/mass_store_run.py +++ b/web/server/codechecker_server/api/mass_store_run.py @@ -47,9 +47,8 @@ from ..database.database import DBSession from ..database.run_db_model import \ AnalysisInfo, AnalyzerStatistic, \ - BugPathEvent, BugReportPoint, \ + ReportPathData, ReportPathDataFile, \ Checker, CheckerSet, CheckerSetItem, \ - ExtendedReportData, \ File, FileContent, \ Report as DBReport, ReportAnnotations, ReviewStatus as ReviewStatusRule, \ Run, RunLock as DBRunLock, RunHistory, \ @@ -61,7 +60,6 @@ from ..session_manager import SessionManager from ..task_executors.abstract_task import AbstractTask, TaskCancelHonoured from ..task_executors.task_manager import TaskManager -from .thrift_enum_helper import report_extended_data_type_str from sqlalchemy.orm import Session as SA_Session @@ -1298,44 +1296,75 @@ def __realise_fake_checkers(self, session): .update({"checker_id": chk_obj.id}, synchronize_session=False) - def __add_report_context(self, session, file_path_to_id): + def __add_report_context( + self, + session: SA_Session, + file_path_to_id: Dict[str, int] + ): for db_report, report in self.__added_reports: + path_data = [] + used_file_ids = set() + LOG.debug("Storing bug path positions.") - for idx, path_pos in enumerate(report.bug_path_positions): - session.add(BugReportPoint( - path_pos.range.start_line, path_pos.range.start_col, - path_pos.range.end_line, path_pos.range.end_col, - idx, file_path_to_id[path_pos.file.path], db_report.id)) + for path_pos in report.bug_path_positions: + path_data.append(ReportPathData.Item( + path_pos.range.start_line, + path_pos.range.start_col, + path_pos.range.end_line, + path_pos.range.end_col, + file_path_to_id[path_pos.file.path], + "path" + )) + used_file_ids.add(file_path_to_id[path_pos.file.path]) LOG.debug("Storing bug path events.") - for idx, event in enumerate(report.bug_path_events): - session.add(BugPathEvent( - event.range.start_line, event.range.start_col, - event.range.end_line, event.range.end_col, - idx, event.message, file_path_to_id[event.file.path], - db_report.id)) + for event in report.bug_path_events: + path_data.append(ReportPathData.Item( + event.range.start_line, + event.range.start_col, + event.range.end_line, + event.range.end_col, + file_path_to_id[event.file.path], + "event", + event.message + )) + used_file_ids.add(file_path_to_id[event.file.path]) LOG.debug("Storing notes.") for note in report.notes: - data_type = report_extended_data_type_str( - ttypes.ExtendedReportDataType.NOTE) - - session.add(ExtendedReportData( - note.range.start_line, note.range.start_col, - note.range.end_line, note.range.end_col, - note.message, file_path_to_id[note.file.path], - db_report.id, data_type)) + path_data.append(ReportPathData.Item( + note.range.start_line, + note.range.start_col, + note.range.end_line, + note.range.end_col, + file_path_to_id[note.file.path], + "note", + note.message + )) + used_file_ids.add(file_path_to_id[note.file.path]) LOG.debug("Storing macro expansions.") for macro in report.macro_expansions: - data_type = report_extended_data_type_str( - ttypes.ExtendedReportDataType.MACRO) + path_data.append(ReportPathData.Item( + macro.range.start_line, + macro.range.start_col, + macro.range.end_line, + macro.range.end_col, + file_path_to_id[macro.file.path], + "macro", + macro.message + )) + used_file_ids.add(file_path_to_id[macro.file.path]) + + report_path_data = ReportPathData(db_report.id, path_data) + session.add(report_path_data) + session.flush() - session.add(ExtendedReportData( - macro.range.start_line, macro.range.start_col, - macro.range.end_line, macro.range.end_col, - macro.message, file_path_to_id[macro.file.path], - db_report.id, data_type)) + rpdf = ReportPathDataFile.insert().values([{ + "report_path_data_id": report_path_data.report_id, + "file_id": fid } for fid in used_file_ids + ]) + session.execute(rpdf) if report.annotations: self.__validate_and_add_report_annotations( @@ -1508,7 +1537,7 @@ def __check_report_count(self): def __store_reports( self, - session: DBSession, + session: SA_Session, report_dir: Path, source_root: Path, run_id: int, diff --git a/web/server/codechecker_server/api/report_server.py b/web/server/codechecker_server/api/report_server.py index 4ef6e4742f..444b68156f 100644 --- a/web/server/codechecker_server/api/report_server.py +++ b/web/server/codechecker_server/api/report_server.py @@ -27,14 +27,14 @@ import sqlalchemy from sqlalchemy.sql.expression import or_, and_, not_, func, \ asc, desc, union_all, select, bindparam, literal_column, cast, true -from sqlalchemy.orm import contains_eager +from sqlalchemy.orm import contains_eager, Session as SA_Session from sqlalchemy.types import ARRAY, String import codechecker_api_shared from codechecker_api.codeCheckerDBAccess_v6 import constants, ttypes from codechecker_api.codeCheckerDBAccess_v6.ttypes import \ AnalysisInfoFilter, AnalysisInfoChecker as API_AnalysisInfoChecker, \ - BlameData, BlameInfo, BugPathPos, \ + BlameData, BlameInfo, \ CheckerCount, CheckerStatusVerificationDetail, Commit, CommitAuthor, \ CommentData, \ DetectionStatus, DiffType, \ @@ -66,12 +66,11 @@ from ..database.run_db_model import \ AnalysisInfo, \ AnalyzerStatistic, \ - BugPathEvent, BugReportPoint, \ - CleanupPlan, CleanupPlanReportHash, Checker, \ - CheckerSetItem, Comment, ExtendedReportData, \ + CleanupPlan, CleanupPlanReportHash, Checker, CheckerSetItem, Comment, \ File, FileContent, \ - Report, ReportAnnotations, ReportAnalysisInfo, ReviewStatus, \ - Run, RunHistory, RunHistoryAnalysisInfo, RunLock, \ + Report, ReportAnnotations, ReportAnalysisInfo, \ + ReportPathData, ReportPathDataFile, \ + ReviewStatus, Run, RunHistory, RunHistoryAnalysisInfo, RunLock, \ SourceComponent, SourceComponentFile, FilterPreset from .common import exc_to_thrift_reqfail @@ -232,7 +231,7 @@ def update_source_component_files( def process_report_filter( - session, + session: SA_Session, run_ids, report_filter, cmp_data=None, @@ -561,57 +560,43 @@ def get_source_component_file_query( def get_reports_by_bugpath_filter_for_single_origin( - session, + session: SA_Session, file_filter_q ) -> Set[int]: """ This function returns a query for report IDs that are fully contained within the files specified by the file_filter_q query.""" - LOG.info("get_reports_by_bugpath_filter file_filter_q: %s", file_filter_q) q_report = session.query(Report.id) \ .join(File, File.id == Report.file_id) \ .filter(file_filter_q) - q_bugpathevent = session.query(BugPathEvent.report_id) \ - .join(File, File.id == BugPathEvent.file_id) \ - .filter(file_filter_q) - - q_bugreportpoint = session.query(BugReportPoint.report_id) \ - .join(File, File.id == BugReportPoint.file_id) \ + q_reportpathdata = session.query(ReportPathData.report_id) \ + .join(ReportPathDataFile, + ReportPathData.report_id == + ReportPathDataFile.c.report_path_data_id) \ + .join(File, File.id == ReportPathDataFile.c.file_id) \ .filter(file_filter_q) - q_extendedreportdata = session.query(ExtendedReportData.report_id) \ - .join(File, File.id == ExtendedReportData.file_id) \ + neg_q_reportpathdata = session.query(ReportPathData.report_id) \ + .join(ReportPathDataFile, + ReportPathData.report_id == + ReportPathDataFile.c.report_path_data_id) \ + .join(File, File.id != ReportPathDataFile.c.file_id) \ .filter(file_filter_q) neg_q_report = session.query(Report.id) \ .join(File, File.id != Report.file_id) \ .filter(file_filter_q) - neg_q_bugpathevent = session.query(BugPathEvent.report_id) \ - .join(File, File.id != BugPathEvent.file_id) \ - .filter(file_filter_q) + return q_report.union(q_reportpathdata) \ + .except_(neg_q_report, neg_q_reportpathdata) - neg_q_bugreportpoint = session.query(BugReportPoint.report_id) \ - .join(File, File.id != BugReportPoint.file_id) \ - .filter(file_filter_q) - neg_q_extendedreportdata = session.query(ExtendedReportData.report_id) \ - .join(File, File.id != ExtendedReportData.file_id) \ - .filter(file_filter_q) - - return q_report.union( - q_bugpathevent, - q_bugreportpoint, - q_extendedreportdata).except_( - neg_q_report, - neg_q_bugpathevent, - neg_q_bugreportpoint, - neg_q_extendedreportdata) - - -def get_reports_by_bugpath_filter(session, file_filter_q) -> Set[int]: +def get_reports_by_bugpath_filter( + session: DBSession, + file_filter_q +) -> Set[int]: """ This function returns a query for report IDs that are related to any file described by the query in the second parameter, either because their bug @@ -622,25 +607,17 @@ def get_reports_by_bugpath_filter(session, file_filter_q) -> Set[int]: .join(File, File.id == Report.file_id) \ .filter(file_filter_q) - q_bugpathevent = session.query(BugPathEvent.report_id) \ - .join(File, File.id == BugPathEvent.file_id) \ - .filter(file_filter_q) - - q_bugreportpoint = session.query(BugReportPoint.report_id) \ - .join(File, File.id == BugReportPoint.file_id) \ - .filter(file_filter_q) - - q_extendedreportdata = session.query(ExtendedReportData.report_id) \ - .join(File, File.id == ExtendedReportData.file_id) \ + q_reportpathdata = session.query(ReportPathData.report_id) \ + .join(ReportPathDataFile, + ReportPathData.report_id == + ReportPathDataFile.c.report_path_data_id) \ + .join(File, File.id == ReportPathDataFile.c.file_id) \ .filter(file_filter_q) - return q_report.union( - q_bugpathevent, - q_extendedreportdata, - q_bugreportpoint) + return q_report.union(q_reportpathdata) -def get_reports_by_components(session, +def get_reports_by_components(session: SA_Session, component_names: List[str], single_origin: bool) -> Set[int]: """ @@ -897,52 +874,38 @@ def process_run_filter(session, query, run_filter): return query -def get_report_details(session, report_ids): +def get_report_details( + session: DBSession, + report_ids: List[int] +) -> Dict[int, ReportDetails]: """ Returns report details for the given report ids. """ details = {} - # Get bug path events. - bug_path_events = session.query(BugPathEvent, File.filepath) \ - .filter(BugPathEvent.report_id.in_(report_ids)) \ - .outerjoin(File, - File.id == BugPathEvent.file_id) \ - .order_by(BugPathEvent.report_id, BugPathEvent.order) - bug_events_list = defaultdict(list) - for event, file_path in bug_path_events: - report_id = event.report_id - event = bugpathevent_db_to_api(event) - event.filePath = file_path - bug_events_list[report_id].append(event) - - # Get bug report points. - bug_report_points = session.query(BugReportPoint, File.filepath) \ - .filter(BugReportPoint.report_id.in_(report_ids)) \ - .outerjoin(File, - File.id == BugReportPoint.file_id) \ - .order_by(BugReportPoint.report_id, BugReportPoint.order) - bug_point_list = defaultdict(list) - for bug_point, file_path in bug_report_points: - report_id = bug_point.report_id - bug_point = bugreportpoint_db_to_api(bug_point) - bug_point.filePath = file_path - bug_point_list[report_id].append(bug_point) - - # Get extended report data. extended_data_list = defaultdict(list) - q = session.query(ExtendedReportData, File.filepath) \ - .filter(ExtendedReportData.report_id.in_(report_ids)) \ - .outerjoin(File, - File.id == ExtendedReportData.file_id) - for data, file_path in q: - report_id = data.report_id - extended_data = extended_data_db_to_api(data) - extended_data.filePath = file_path - extended_data_list[report_id].append(extended_data) + report_path_data = session.query(ReportPathData) \ + .filter(ReportPathData.report_id.in_(report_ids)) + + for rpd in report_path_data: + files = {f.id: f.filepath for f in rpd.files} + + for pd in rpd.path_data: + if pd.type == "event": + event = bugpathevent_db_to_api(pd) + event.filePath = files[pd.file_id] + bug_events_list[rpd.report_id].append(event) + elif pd.type == "path": + bug_point = bugreportpoint_db_to_api(pd) + bug_point.filePath = files[pd.file_id] + bug_point_list[rpd.report_id].append(bug_point) + else: + extended_data = extended_data_db_to_api(pd) + extended_data.filePath = files[pd.file_id] + extended_data_list[rpd.report_id].append(extended_data) # Get Comments for report data comment_data_list = defaultdict(list) @@ -965,33 +928,35 @@ def get_report_details(session, report_ids): return details -def bugpathevent_db_to_api(bpe): +def bugpathevent_db_to_api(bpe: ReportPathData.Item) -> ttypes.BugPathEvent: return ttypes.BugPathEvent( - startLine=bpe.line_begin, - startCol=bpe.col_begin, - endLine=bpe.line_end, - endCol=bpe.col_end, + startLine=bpe.from_row, + startCol=bpe.from_col, + endLine=bpe.to_row, + endCol=bpe.to_col, msg=bpe.msg, fileId=bpe.file_id) -def bugreportpoint_db_to_api(brp): - return BugPathPos( - startLine=brp.line_begin, - startCol=brp.col_begin, - endLine=brp.line_end, - endCol=brp.col_end, +def bugreportpoint_db_to_api(brp: ReportPathData.Item) -> ttypes.BugPathPos: + return ttypes.BugPathPos( + startLine=brp.to_row, + startCol=brp.from_col, + endLine=brp.to_row, + endCol=brp.to_col, fileId=brp.file_id) -def extended_data_db_to_api(erd): +def extended_data_db_to_api( + erd: ReportPathData.Item +) -> ttypes.ExtendedReportData: return ttypes.ExtendedReportData( type=report_extended_data_type_enum(erd.type), - startLine=erd.line_begin, - startCol=erd.col_begin, - endLine=erd.line_end, - endCol=erd.col_end, - message=erd.message, + startLine=erd.from_row, + startCol=erd.from_col, + endLine=erd.to_row, + endCol=erd.to_col, + message=erd.msg, fileId=erd.file_id) diff --git a/web/server/codechecker_server/database/db_cleanup.py b/web/server/codechecker_server/database/db_cleanup.py index fe4e1c5a8e..7925648b27 100644 --- a/web/server/codechecker_server/database/db_cleanup.py +++ b/web/server/codechecker_server/database/db_cleanup.py @@ -22,11 +22,11 @@ from .database import DBSession from .run_db_model import \ AnalysisInfo, \ - BugPathEvent, BugReportPoint, \ CheckerSet, \ Comment, Checker, \ File, FileContent, \ - Report, ReportAnalysisInfo, RunHistoryAnalysisInfo, RunLock + Report, ReportAnalysisInfo, ReportPathDataFile, RunHistoryAnalysisInfo, \ + RunLock LOG = get_logger('server') RUN_LOCK_TIMEOUT_IN_DATABASE = 30 * 60 # 30 minutes. @@ -90,19 +90,16 @@ def remove_unused_files(product): LOG.debug("[%s] Garbage collection of dangling files started...", product.endpoint) try: - bpe_files = session.query(BugPathEvent.file_id) \ - .group_by(BugPathEvent.file_id) - brp_files = session.query(BugReportPoint.file_id) \ - .group_by(BugReportPoint.file_id) + files = session.query(ReportPathDataFile.c.file_id) \ + .group_by(ReportPathDataFile.c.file_id) files_to_delete = session.query(File.id) \ - .filter(File.id.notin_(bpe_files), File.id.notin_(brp_files)) + .filter(File.id.notin_(files)) files_to_delete = map(lambda x: x[0], files_to_delete) total_count = 0 for chunk in util.chunks(iter(files_to_delete), chunk_size): - q = session.query(File) \ - .filter(File.id.in_(chunk)) + q = session.query(File).filter(File.id.in_(chunk)) count = q.delete(synchronize_session=False) if count: total_count += count diff --git a/web/server/codechecker_server/database/run_db_model.py b/web/server/codechecker_server/database/run_db_model.py index ea70e4c4e0..054fb2cbd2 100644 --- a/web/server/codechecker_server/database/run_db_model.py +++ b/web/server/codechecker_server/database/run_db_model.py @@ -8,19 +8,21 @@ """ SQLAlchemy ORM model for the analysis run storage database. """ +from dataclasses import dataclass from datetime import datetime, timedelta from math import ceil import os import json import hashlib from typing import Optional, List +import zlib from sqlalchemy import BigInteger, Boolean, Column, DateTime, Enum, \ ForeignKey, Integer, LargeBinary, MetaData, String, UniqueConstraint, \ Table, Text, JSON, case from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import true, false CC_META = MetaData(naming_convention={ @@ -309,106 +311,92 @@ def __init__(self, filepath, content_hash, remote_url, tracking_branch): self.tracking_branch = tracking_branch -class BugPathEvent(Base): - __tablename__ = 'bug_path_events' - - line_begin = Column(Integer) - col_begin = Column(Integer) - line_end = Column(Integer) - col_end = Column(Integer) - - order = Column(Integer, primary_key=True) - - msg = Column(String) - file_id = Column(Integer, ForeignKey('files.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), index=True) - report_id = Column(BigInteger, ForeignKey('reports.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), - index=True, - primary_key=True) - - def __init__(self, line_begin, col_begin, line_end, col_end, - order, msg, file_id, report_id): - self.line_begin, self.col_begin, self.line_end, self.col_end = \ - line_begin, col_begin, line_end, col_end - - self.order = order - self.msg = msg - self.file_id = file_id - self.report_id = report_id - - -class BugReportPoint(Base): - __tablename__ = 'bug_report_points' - - line_begin = Column(Integer) - col_begin = Column(Integer) - line_end = Column(Integer) - col_end = Column(Integer) - - order = Column(Integer, primary_key=True) - - file_id = Column(Integer, ForeignKey('files.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), index=True) - report_id = Column(BigInteger, ForeignKey('reports.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), - index=True, - primary_key=True) - - def __init__(self, line_begin, col_begin, line_end, col_end, - order, file_id, report_id): - self.line_begin, self.col_begin, self.line_end, self.col_end = \ - line_begin, col_begin, line_end, col_end - - self.order = order - self.file_id = file_id - self.report_id = report_id +ReportPathDataFile = Table( + 'report_path_data_files', + Base.metadata, + Column( + 'report_path_data_id', + BigInteger, + ForeignKey('report_path_data.report_id', + deferrable=True, + initially="DEFERRED", + ondelete="CASCADE"), + index=True), + Column( + 'file_id', + Integer, + ForeignKey('files.id', + deferrable=True, + initially="DEFERRED", + ondelete="CASCADE"), + index=True) +) -class ExtendedReportData(Base): +class ReportPathData(Base): """ - Store extra information which can help to understand or fix a report. + Store bug path positions, bug events, macro expansions, fix-its, etc. """ - __tablename__ = 'extended_report_data' + __tablename__ = 'report_path_data' - id = Column(Integer, autoincrement=True, primary_key=True) - - report_id = Column(BigInteger, ForeignKey('reports.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), - index=True) - - file_id = Column(Integer, ForeignKey('files.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), index=True) + report_id = Column( + BigInteger, + ForeignKey( + 'reports.id', + deferrable=True, + initially="DEFERRED", + ondelete='CASCADE'), + primary_key=True) - type = Column(Enum('note', - 'macro', - 'fixit', - name='extended_data_type')) + @dataclass + class Item: + from_row: int + from_col: int + to_row: int + to_col: int + file_id: int + type: str + msg: Optional[str] = None + + def to_dict(self): + return { + "from": { "row": self.from_row, "col": self.from_col }, + "to": { "row": self.to_row, "col": self.to_col }, + "fid": self.file_id, + "type": self.type, + **({"msg": self.msg} if self.msg is not None else {}) + } + + @classmethod + def from_dict(cls, data: dict): + return cls( + data["from"]["row"], + data["from"]["col"], + data["to"]["row"], + data["to"]["col"], + data["fid"], + data["type"], + data.get("msg")) + + _path_data = Column("path_data", LargeBinary) - line_begin = Column(Integer) - col_begin = Column(Integer) - line_end = Column(Integer) - col_end = Column(Integer) + @hybrid_property + def path_data(self): + pd = json.loads(zlib.decompress(self._path_data).decode("utf-8")) + return list(map(ReportPathData.Item.from_dict, pd)) - message = Column(String) + @path_data.setter + def path_data(self, data: List[Item]): + self._path_data = zlib.compress( + json.dumps( + list(map(ReportPathData.Item.to_dict, data))).encode("utf-8"), + zlib.Z_BEST_COMPRESSION) - def __init__(self, line_begin, col_begin, line_end, col_end, - message, file_id, report_id, data_type): + files = relationship("File", secondary=ReportPathDataFile) - self.line_begin = line_begin - self.col_begin = col_begin - self.line_end = line_end - self.col_end = col_end - self.message = message - self.file_id = file_id + def __init__(self, report_id, path_data): self.report_id = report_id - self.type = data_type + self.path_data = path_data ReportAnalysisInfo = Table( diff --git a/web/server/codechecker_server/migrations/report/versions/6c3c93a826f4_report_path_data_table.py b/web/server/codechecker_server/migrations/report/versions/6c3c93a826f4_report_path_data_table.py new file mode 100644 index 0000000000..241f6f5794 --- /dev/null +++ b/web/server/codechecker_server/migrations/report/versions/6c3c93a826f4_report_path_data_table.py @@ -0,0 +1,324 @@ +""" +Report path data table + +Revision ID: 6c3c93a826f4 +Revises: b1be74589382 +Create Date: 2026-07-28 16:10:04.503407 +""" + +from logging import getLogger + +from alembic import op +import sqlalchemy as sa + +import json +import zlib +from collections import defaultdict + +from codechecker_common import util + + +# Revision identifiers, used by Alembic. +revision = '6c3c93a826f4' +down_revision = 'b1be74589382' +branch_labels = None +depends_on = None + + +def upgrade(): + LOG = getLogger("migration/report") + # ### commands auto generated by Alembic - please adjust! ### + report_path_data_table = op.create_table( + 'report_path_data', + sa.Column('report_id', sa.BigInteger(), nullable=False), + sa.Column('path_data', sa.LargeBinary(), nullable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_report_path_data_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint('report_id', name=op.f('pk_report_path_data')) + ) + report_path_data_files_table = op.create_table( + 'report_path_data_files', + sa.Column('report_path_data_id', sa.BigInteger(), nullable=True), + sa.Column('file_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_report_path_data_files_file_id_files')), + sa.ForeignKeyConstraint( + ['report_path_data_id'], + ['report_path_data.report_id'], + name=op.f( + 'fk_report_path_data_files_report_path_data_id_' + 'report_path_data'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True) + ) + + conn = op.get_bind() + + report_ids = conn.execute(sa.text("""SELECT id FROM reports""")) + + for report_id_chunk in util.chunks(report_ids, 100_000): + report_id_chunk = list(map(lambda x: x[0], report_id_chunk)) + + conn.execute(sa.text("BEGIN")) + + report_path_data = defaultdict(lambda: { + 'path_data': [], + 'used_file_ids': set() + }) + + bug_report_points = conn.execute(sa.text(f""" + SELECT * FROM bug_report_points + WHERE report_id IN ({','.join(map(str, report_id_chunk))}) + ORDER BY report_id, "order" + """)) + + for brp in bug_report_points: + report_path_data[brp.report_id]['path_data'].append({ + "from": { + "row": brp.line_begin, + "col": brp.col_begin + }, + "to": { + "row": brp.line_end, + "col": brp.col_end + }, + "type": "path", + "fid": brp.file_id + }) + report_path_data[brp.report_id]['used_file_ids'].add(brp.file_id) + + bug_path_events = conn.execute(sa.text(f""" + SELECT * FROM bug_path_events + WHERE report_id IN ({','.join(map(str, report_id_chunk))}) + ORDER BY report_id, "order" + """)) + + for bpe in bug_path_events: + report_path_data[bpe.report_id]['path_data'].append({ + "from": { + "row": bpe.line_begin, + "col": bpe.col_begin + }, + "to": { + "row": bpe.line_end, + "col": bpe.col_end + }, + "type": "event", + "msg": bpe.msg, + "fid": bpe.file_id + }) + report_path_data[bpe.report_id]['used_file_ids'].add(bpe.file_id) + + extended_report_data = conn.execute(sa.text(f""" + SELECT * FROM extended_report_data + WHERE report_id in ({','.join(map(str, report_id_chunk))}) + """)) + + for erd in extended_report_data: + report_path_data[erd.report_id]['path_data'].append({ + "from": { + "row": erd.line_begin, + "col": erd.col_begin + }, + "to": { + "row": erd.line_end, + "col": erd.col_end + }, + "type": erd.type, + "msg": erd.message, + "fid": erd.file_id + }) + report_path_data[erd.report_id]['used_file_ids'].add(erd.file_id) + + report_data_content = [] + report_data_file_content = [] + + for report_id, data in report_path_data.items(): + compressed_data = zlib.compress( + json.dumps(data['path_data']).encode('utf-8'), + zlib.Z_BEST_COMPRESSION) + + report_data_content.append({ + 'report_id': report_id, + 'path_data': compressed_data + }) + + report_data_file_content.extend([{ + 'report_path_data_id': report_id, + 'file_id': fid + } for fid in data['used_file_ids']]) + + conn.execute( + sa.insert(report_path_data_table), + report_data_content) + + conn.execute( + sa.insert(report_path_data_files_table), + report_data_file_content) + + conn.execute(sa.text("COMMIT")) + + op.create_index( + op.f('ix_report_path_data_files_report_path_data_id'), + 'report_path_data_files', + ['report_path_data_id'], + unique=False) + op.drop_index( + op.f('ix_bug_report_points_file_id'), + table_name='bug_report_points') + op.drop_index( + op.f('ix_bug_report_points_report_id'), + table_name='bug_report_points') + op.drop_table('bug_report_points') + op.drop_index( + op.f('ix_bug_path_events_file_id'), + table_name='bug_path_events') + op.drop_index( + op.f('ix_bug_path_events_report_id'), + table_name='bug_path_events') + op.drop_table('bug_path_events') + op.drop_index( + op.f('ix_extended_report_data_file_id'), + table_name='extended_report_data') + op.drop_index( + op.f('ix_extended_report_data_report_id'), + table_name='extended_report_data') + op.drop_table('extended_report_data') + + # ### end Alembic commands ### + + +def downgrade(): + # TODO: Currently we're not supporting the downgrade of DB schema version. + # This is a script generated by Alembic, but the old tables are not + # populated by data from the new schema. + LOG = getLogger("migration/report") + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'extended_report_data', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('report_id', sa.INTEGER(), nullable=False), + sa.Column('file_id', sa.INTEGER(), nullable=True), + sa.Column('type', sa.VARCHAR(length=5), nullable=True), + sa.Column('line_begin', sa.INTEGER(), nullable=True), + sa.Column('col_begin', sa.INTEGER(), nullable=True), + sa.Column('line_end', sa.INTEGER(), nullable=True), + sa.Column('col_end', sa.INTEGER(), nullable=True), + sa.Column('message', sa.VARCHAR(), nullable=True), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_extended_report_data_file_id_files'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_extended_report_data_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_extended_report_data')) + ) + op.create_index( + op.f('ix_extended_report_data_report_id'), + 'extended_report_data', + ['report_id'], + unique=False) + op.create_index( + op.f('ix_extended_report_data_file_id'), + 'extended_report_data', + ['file_id'], + unique=False) + op.create_table( + 'bug_path_events', + sa.Column('line_begin', sa.INTEGER(), nullable=True), + sa.Column('col_begin', sa.INTEGER(), nullable=True), + sa.Column('line_end', sa.INTEGER(), nullable=True), + sa.Column('col_end', sa.INTEGER(), nullable=True), + sa.Column('order', sa.INTEGER(), nullable=False), + sa.Column('msg', sa.VARCHAR(), nullable=True), + sa.Column('file_id', sa.INTEGER(), nullable=True), + sa.Column('report_id', sa.INTEGER(), nullable=False), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_bug_path_events_file_id_files'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_bug_path_events_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint( + 'order', + 'report_id', + name=op.f('pk_bug_path_events')) + ) + op.create_index( + op.f('ix_bug_path_events_report_id'), + 'bug_path_events', + ['report_id'], + unique=False) + op.create_index( + op.f('ix_bug_path_events_file_id'), + 'bug_path_events', + ['file_id'], + unique=False) + op.create_table( + 'bug_report_points', + sa.Column('line_begin', sa.INTEGER(), nullable=True), + sa.Column('col_begin', sa.INTEGER(), nullable=True), + sa.Column('line_end', sa.INTEGER(), nullable=True), + sa.Column('col_end', sa.INTEGER(), nullable=True), + sa.Column('order', sa.INTEGER(), nullable=False), + sa.Column('file_id', sa.INTEGER(), nullable=True), + sa.Column('report_id', sa.INTEGER(), nullable=False), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_bug_report_points_file_id_files'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_bug_report_points_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint( + 'order', + 'report_id', + name=op.f('pk_bug_report_points')) + ) + op.create_index( + op.f('ix_bug_report_points_report_id'), + 'bug_report_points', + ['report_id'], + unique=False) + op.create_index( + op.f('ix_bug_report_points_file_id'), + 'bug_report_points', + ['file_id'], + unique=False) + op.drop_index( + op.f('ix_report_path_data_files_report_path_data_id'), + table_name='report_path_data_files') + op.drop_table('report_path_data_files') + op.drop_table('report_path_data') + # ### end Alembic commands ###