"""This module handles all operations involving fetching reports. The process of fetching reports is made up of these steps: 1. Each vendor is queried for its supported reports using the SUSHI API 2. The vendor is then queried for each supported report, also using the SUSHI API 3. The raw JSON response is converted to JsonModel objects that make it easier to work with the JSON data. 4. The model objects are then used to create create row objects that will be in the final TSV file. 5. The row objects are then sorted by their primary columns, for example, item reports are sorted by the item column. 6. The sorted rows are then used to create and save a final TSV report file that adheres to the COUNTER 5 standards. 7. After all reports are processed, the database is updated with the new data .. NOTE:: All fetch operations are multi-threaded. Each vendor has it's own thread, each report for that vendor also has it's own thread. The maximum concurrent vendors and reports (per vendor) can be changed in the settings tab on the GUI """ from os import path, makedirs import csv import json import requests import platform import copy import ctypes from PyQt5.QtCore import QObject, QThread, pyqtSignal, QDate, Qt from PyQt5.QtGui import QStandardItemModel, QStandardItem from PyQt5.QtWidgets import QPushButton, QDialog, QWidget, QProgressBar, QLabel, QVBoxLayout, QDialogButtonBox, \ QCheckBox, QDateEdit, QFrame, QHBoxLayout, QSizePolicy, QLineEdit, QListView, QRadioButton, QButtonGroup import GeneralUtils from ui import FetchReportsTab, FetchSpecialReportsTab, FetchProgressDialog, ReportResultWidget, VendorResultsWidget from GeneralUtils import JsonModel from ManageVendors import Vendor from Settings import SettingsModel from ManageDB import UpdateDatabaseWorker from Constants import * # region Models class SupportedReportModel(JsonModel): """Models a SUSHI Supported Report""" def __init__(self, report_id: str): self.report_id = report_id @classmethod def from_json(cls, json_dict: dict): report_id = str(json_dict["Report_ID"]).upper() if "Report_ID" in json_dict else "" return cls(report_id) class PeriodModel(JsonModel): """Models a SUSHI Period""" def __init__(self, begin_date: str, end_date: str): self.begin_date = begin_date self.end_date = end_date @classmethod def from_json(cls, json_dict: dict): begin_date = json_dict["Begin_Date"] if "Begin_Date" in json_dict else "" end_date = json_dict["End_Date"] if "End_Date" in json_dict else "" return cls(begin_date, end_date) class InstanceModel(JsonModel): """Models a SUSHI Instance""" def __init__(self, metric_type: str, count: int): self.metric_type = metric_type self.count = count @classmethod def from_json(cls, json_dict: dict): metric_type = json_dict["Metric_Type"] if "Metric_Type" in json_dict else "" count = int(json_dict["Count"]) if "Count" in json_dict else 0 return cls(metric_type, count) class PerformanceModel(JsonModel): """Models a SUSHI Performance""" def __init__(self, period: PeriodModel, instances: list): self.period = period self.instances = instances @classmethod def from_json(cls, json_dict: dict): period = PeriodModel.from_json(json_dict["Period"]) if "Period" in json_dict else None instances = get_models("Instance", InstanceModel, json_dict) return cls(period, instances) class TypeValueModel(JsonModel): """Models SUSHI models that are made up of Type and value""" def __init__(self, item_type: str, value: str): self.item_type = item_type self.value = value @classmethod def from_json(cls, json_dict: dict): item_type = str(json_dict["Type"]) if "Type" in json_dict else "" value = str(json_dict["Value"]) if "Value" in json_dict else "" return cls(item_type, value) class NameValueModel(JsonModel): """Models SUSHI models that are made up of Name and value""" def __init__(self, name: str, value: str): self.name = name self.value = value @classmethod def from_json(cls, json_dict: dict): name = str(json_dict["Name"]) if "Name" in json_dict else "" value = str(json_dict["Value"]) if "Value" in json_dict else "" return cls(name, value) class ExceptionModel(JsonModel): """Models a SUSHI Exception""" def __init__(self, code: int, message: str, severity: str, data: str): self.code = code self.message = message self.severity = severity self.data = data @classmethod def from_json(cls, json_dict: dict): code = int(json_dict["Code"]) if "Code" in json_dict else 0 message = json_dict["Message"] if "Message" in json_dict else "" severity = json_dict["Severity"] if "Severity" in json_dict else "" data = json_dict["Data"] if "Data" in json_dict else "" return cls(code, message, severity, data) class ReportHeaderModel(JsonModel): """Models a SUSHI Report Header""" def __init__(self, report_name: str, report_id: str, release: str, institution_name: str, institution_ids: list, report_filters: list, report_attributes: list, exceptions: list, created: str, created_by: str): self.report_name = report_name self.report_id = report_id self.release = release self.institution_name = institution_name self.institution_ids = institution_ids self.report_filters = report_filters self.report_attributes = report_attributes self.exceptions = exceptions self.created = created self.created_by = created_by # Not part of JSON self.major_report_type = None @classmethod def from_json(cls, json_dict: dict): report_name = json_dict["Report_Name"] if "Report_Name" in json_dict else "" report_id = str(json_dict["Report_ID"]).upper() if "Report_ID" in json_dict else "" release = json_dict["Release"] if "Release" in json_dict else "" institution_name = json_dict["Institution_Name"] if "Institution_Name" in json_dict else "" created = json_dict["Created"] if "Created" in json_dict else "" created_by = json_dict["Created_By"] if "Created_By" in json_dict else "" institution_ids = get_models("Institution_ID", TypeValueModel, json_dict) report_filters = get_models("Report_Filters", NameValueModel, json_dict) report_attributes = get_models("Report_Attributes", NameValueModel, json_dict) exceptions = get_models("Exceptions", ExceptionModel, json_dict) return cls(report_name, report_id, release, institution_name, institution_ids, report_filters, report_attributes, exceptions, created, created_by) class ReportModel(JsonModel): """Models a SUSHI Report""" def __init__(self, report_header: ReportHeaderModel, report_items: list): self.report_header = report_header self.report_items = report_items # Not part of JSON self.exceptions = [] @classmethod def from_json(cls, json_dict: dict): exceptions = ReportModel.process_exceptions(json_dict) report_header = ReportHeaderModel.from_json(json_dict["Report_Header"]) report_type = report_header.report_id major_report_type = GeneralUtils.get_major_report_type(report_type) report_header.major_report_type = major_report_type report_items = [] if "Report_Items" in json_dict: report_item_dicts = json_dict["Report_Items"] if len(report_item_dicts) > 0: if major_report_type == MajorReportType.PLATFORM: for report_item_dict in report_item_dicts: report_items.append(PlatformReportItemModel.from_json(report_item_dict)) elif major_report_type == MajorReportType.DATABASE: for report_item_dict in report_item_dicts: report_items.append(DatabaseReportItemModel.from_json(report_item_dict)) elif major_report_type == MajorReportType.TITLE: for report_item_dict in report_item_dicts: report_items.append(TitleReportItemModel.from_json(report_item_dict)) elif major_report_type == MajorReportType.ITEM: for report_item_dict in report_item_dicts: report_items.append(ItemReportItemModel.from_json(report_item_dict)) report_model = cls(report_header, report_items) report_model.exceptions = exceptions return report_model @classmethod def process_exceptions(cls, json_dict: dict) -> list: """Gets all exception models in a JSON dict, returns them as a list :param json_dict: A JSON dict :raises ReportHeaderMissingException: When the report header is missing :raises RetryLaterException: When a retry later exception model is received :raises UnacceptableCodeException: When the report cannot be processed based on the exception code """ exceptions = [] if "Exception" in json_dict: exceptions.append(ExceptionModel.from_json(json_dict["Exception"])) code = int(json_dict["Code"]) if "Code" in json_dict else "" message = json_dict["Message"] if "Message" in json_dict else "" data = json_dict["Data"] if "Data" in json_dict else "" severity = json_dict["Severity"] if "Severity" in json_dict else "" if code: exceptions.append(ExceptionModel(code, message, severity, data)) if "Report_Header" in json_dict: report_header = ReportHeaderModel.from_json(json_dict["Report_Header"]) if len(report_header.exceptions) > 0: for exception in report_header.exceptions: exceptions.append(exception) else: raise ReportHeaderMissingException(exceptions) for exception in exceptions: if exception.code in RETRY_LATER_CODES: raise RetryLaterException(exceptions) elif exception.code not in ACCEPTABLE_CODES: raise UnacceptableCodeException(exceptions) return exceptions class PlatformReportItemModel(JsonModel): """Models a SUSHI Platform Report Item""" def __init__(self, platform: str, data_type: str, access_method: str, performances: list): self.platform = platform self.data_type = data_type self.access_method = access_method self.performances = performances @classmethod def from_json(cls, json_dict: dict): platform = json_dict["Platform"] if "Platform" in json_dict else "" data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else "" access_method = json_dict["Access_Method"] if "Access_Method" in json_dict else "" performances = get_models("Performance", PerformanceModel, json_dict) return cls(platform, data_type, access_method, performances) class DatabaseReportItemModel(JsonModel): """Models a SUSHI Database Report Item""" def __init__(self, database: str, publisher: str, item_ids: list, publisher_ids: list, platform: str, data_type: str, access_method: str, performances: list): self.database = database self.publisher = publisher self.item_ids = item_ids self.publisher_ids = publisher_ids self.platform = platform self.data_type = data_type self.access_method = access_method self.performances = performances @classmethod def from_json(cls, json_dict: dict): database = json_dict["Database"] if "Database" in json_dict else "" publisher = json_dict["Publisher"] if "Publisher" in json_dict else "" platform = json_dict["Platform"] if "Platform" in json_dict else "" data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else "" access_method = json_dict["Access_Method"] if "Access_Method" in json_dict else "" item_ids = get_models("Item_ID", TypeValueModel, json_dict) publisher_ids = get_models("Publisher_ID", TypeValueModel, json_dict) performances = get_models("Performance", PerformanceModel, json_dict) return cls(database, publisher, item_ids, publisher_ids, platform, data_type, access_method, performances) class TitleReportItemModel(JsonModel): """Models a SUSHI Title Report Item""" def __init__(self, title: str, item_ids: list, platform: str, publisher: str, publisher_ids: list, data_type: str, section_type: str, yop: str, access_type: str, access_method: str, performances: list): self.title = title self.item_ids = item_ids self.platform = platform self.publisher = publisher self.publisher_ids = publisher_ids self.data_type = data_type self.section_type = section_type self.yop = yop # Year of publication self.access_type = access_type self.access_method = access_method self.performances = performances @classmethod def from_json(cls, json_dict: dict): title = json_dict["Title"] if "Title" in json_dict else "" platform = json_dict["Platform"] if "Platform" in json_dict else "" publisher = json_dict["Publisher"] if "Publisher" in json_dict else "" data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else "" section_type = json_dict["Section_Type"] if "Section_Type" in json_dict else "" yop = json_dict["YOP"] if "YOP" in json_dict else "" access_type = json_dict["Access_Type"] if "Access_Type" in json_dict else "" access_method = json_dict["Access_Method"] if "Access_Method" in json_dict else "" item_ids = get_models("Item_ID", TypeValueModel, json_dict) publisher_ids = get_models("Publisher_ID", TypeValueModel, json_dict) performances = get_models("Performance", PerformanceModel, json_dict) return cls(title, item_ids, platform, publisher, publisher_ids, data_type, section_type, yop, access_type, access_method, performances) class ItemContributorModel(JsonModel): """Models a SUSHI Item Contributor""" def __init__(self, item_type: str, name: str, identifier: str): self.item_type = item_type self.name = name self.identifier = identifier @classmethod def from_json(cls, json_dict: dict): item_type = json_dict["Type"] if "Type" in json_dict else "" name = json_dict["Name"] if "Name" in json_dict else "" identifier = json_dict["Identifier"] if "Identifier" in json_dict else "" return cls(item_type, name, identifier) class ItemParentModel(JsonModel): """Models a SUSHI Item Parent""" def __init__(self, item_name: str, item_ids: list, item_contributors: list, item_dates: list, item_attributes: list, data_type: str): self.item_name = item_name self.item_ids = item_ids self.item_contributors = item_contributors self.item_dates = item_dates self.item_attributes = item_attributes self.data_type = data_type @classmethod def from_json(cls, json_dict: dict): item_name = json_dict["Item_Name"] if "Item_Name" in json_dict else "" data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else "" item_ids = get_models("Item_ID", TypeValueModel, json_dict) item_contributors = get_models("Item_Contributors", ItemContributorModel, json_dict) item_dates = get_models("Item_Dates", TypeValueModel, json_dict) item_attributes = get_models("Item_Attributes", TypeValueModel, json_dict) return cls(item_name, item_ids, item_contributors, item_dates, item_attributes, data_type) class ItemComponentModel(JsonModel): """Models a SUSHI Item Component""" def __init__(self, item_name: str, item_ids: list, item_contributors: list, item_dates: list, item_attributes: list, data_type: str, performances: list): self.item_name = item_name self.item_ids = item_ids self.item_contributors = item_contributors self.item_dates = item_dates self.item_attributes = item_attributes self.data_type = data_type self.performances = performances @classmethod def from_json(cls, json_dict: dict): item_name = json_dict["Item_Name"] if "Item_Name" in json_dict else "" data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else "" item_ids = get_models("Item_ID", TypeValueModel, json_dict) item_contributors = get_models("Item_Contributors", ItemContributorModel, json_dict) item_dates = get_models("Item_Dates", TypeValueModel, json_dict) item_attributes = get_models("Item_Attributes", TypeValueModel, json_dict) performances = get_models("Performance", PerformanceModel, json_dict) return cls(item_name, item_ids, item_contributors, item_dates, item_attributes, data_type, performances) class ItemReportItemModel(JsonModel): """Models a SUSHI Item Report model""" def __init__(self, item: str, item_ids: list, item_contributors: list, item_dates: list, item_attributes: list, platform: str, publisher: str, publisher_ids: list, item_parent: ItemParentModel, item_components: list, data_type: str, yop: str, access_type: str, access_method: str, performances: list): self.item = item self.item_ids = item_ids self.item_contributors = item_contributors self.item_dates = item_dates self.item_attributes = item_attributes self.platform = platform self.publisher = publisher self.publisher_ids = publisher_ids self.item_parent = item_parent self.item_components = item_components self.data_type = data_type self.yop = yop # Year of publication self.access_type = access_type self.access_method = access_method self.performances = performances @classmethod def from_json(cls, json_dict: dict): item = json_dict["Item"] if "Item" in json_dict else "" platform = json_dict["Platform"] if "Platform" in json_dict else "" publisher = json_dict["Publisher"] if "Publisher" in json_dict else "" data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else "" yop = json_dict["YOP"] if "YOP" in json_dict else "" access_type = json_dict["Access_Type"] if "Access_Type" in json_dict else "" access_method = json_dict["Access_Method"] if "Access_Method" in json_dict else "" item_parent = ItemParentModel.from_json(json_dict["Item_Parent"]) if "Item_Parent" in json_dict else None item_ids = get_models("Item_ID", TypeValueModel, json_dict) item_contributors = get_models("Item_Contributors", ItemContributorModel, json_dict) item_dates = get_models("Item_Dates", TypeValueModel, json_dict) item_attributes = get_models("Item_Attributes", TypeValueModel, json_dict) publisher_ids = get_models("Publisher_ID", TypeValueModel, json_dict) item_components = get_models("Item_Component", ItemComponentModel, json_dict) performances = get_models("Performance", PerformanceModel, json_dict) return cls(item, item_ids, item_contributors, item_dates, item_attributes, platform, publisher, publisher_ids, item_parent, item_components, data_type, yop, access_type, access_method, performances) # endregion # region Custom Exceptions class RetryLaterException(Exception): """An exception raised when a retry later exception code is received in an exception model""" def __init__(self, exceptions: list): self.exceptions = exceptions class ReportHeaderMissingException(Exception): """An exception raised when a report header is missing from a report""" def __init__(self, exceptions: list): self.exceptions = exceptions class UnacceptableCodeException(Exception): """An exception raised when a report cannot be processed based on an exception code""" def __init__(self, exceptions: list): self.exceptions = exceptions # endregion def exception_models_to_message(exceptions: list) -> str: """Formats a list of exception models into a single string """ message = "" for exception in exceptions: if message: message += "\n\n" message += f"Code: {exception.code}" \ f"\nMessage: {exception.message}" \ f"\nSeverity: {exception.severity}" \ f"\nData: {exception.data}" return message def get_models(model_key: str, model_type, json_dict: dict) -> list: """This converts json lists into a list of the specified SUSHI model type :param model_key: The target key to get the list of JSONObjects :param model_type: The target model type, e.g PerformanceModel :param json_dict: The JSON dict to get the list from """ # Some vendors sometimes return a single dict even when the standard specifies a list, # we need to check for that models = [] if model_key in json_dict and json_dict[model_key] is not None: if type(json_dict[model_key]) is list: model_dicts = json_dict[model_key] for model_dict in model_dicts: models.append(model_type.from_json(model_dict)) elif type(json_dict[model_key]) is dict: models.append(model_type.from_json(json_dict[model_key])) return models def get_month_years(begin_date: QDate, end_date: QDate) -> list: """Returns a list of month-year (MMM-yyyy) strings within a date range""" month_years = [] if begin_date.year() == end_date.year(): num_months = (end_date.month() - begin_date.month()) + 1 else: num_months = (12 - begin_date.month() + end_date.month()) + 1 num_years = end_date.year() - begin_date.year() num_months += (num_years - 1) * 12 for i in range(num_months): month_years.append(begin_date.addMonths(i).toString("MMM-yyyy")) return month_years class SpecialReportOptions: """This holds all the parameters that are used to process a special report The options are stored as tuples, (option has non-default value, option type, option name, list of option values) """ def __init__(self): # PR, DR, TR, IR self.data_type = False, SpecialOptionType.AP, "Data_Type", [DEFAULT_SPECIAL_OPTION_VALUE] self.access_method = False, SpecialOptionType.AP, "Access_Method", [DEFAULT_SPECIAL_OPTION_VALUE] self.metric_type = False, SpecialOptionType.POS, "Metric_Type", [DEFAULT_SPECIAL_OPTION_VALUE] self.exclude_monthly_details = False, SpecialOptionType.TO, None, None # TR, IR current_date = QDate.currentDate() self.yop = False, SpecialOptionType.ADP, "YOP", [DEFAULT_SPECIAL_OPTION_VALUE] self.access_type = False, SpecialOptionType.AP, "Access_Type", [DEFAULT_SPECIAL_OPTION_VALUE] # TR self.section_type = False, SpecialOptionType.AP, "Section_Type", [DEFAULT_SPECIAL_OPTION_VALUE] # IR self.authors = False, SpecialOptionType.AO, "Authors", [DEFAULT_SPECIAL_OPTION_VALUE] self.publication_date = False, SpecialOptionType.AO, "Publication_Date", [DEFAULT_SPECIAL_OPTION_VALUE] self.article_version = False, SpecialOptionType.AO, "Article_Version", [DEFAULT_SPECIAL_OPTION_VALUE] self.include_component_details = False, SpecialOptionType.POB, None, None self.include_parent_details = False, SpecialOptionType.POB, None, None class ReportRow: """This models a row in the generated report, it contains every possible column :param begin_date: The begin date of the request, used to populate the month columns in the report :param end_date: The end date of the request, used to populate the month columns in the report """ def __init__(self, begin_date: QDate, end_date: QDate): self.database = "" self.title = "" self.item = "" self.publisher = "" self.publisher_id = "" self.platform = "" self.authors = "" self.publication_date = "" self.article_version = "" self.doi = "" self.proprietary_id = "" self.online_issn = "" self.print_issn = "" self.linking_issn = "" self.isbn = "" self.uri = "" self.parent_title = "" self.parent_authors = "" self.parent_publication_date = "" self.parent_article_version = "" self.parent_data_type = "" self.parent_doi = "" self.parent_proprietary_id = "" self.parent_online_issn = "" self.parent_print_issn = "" self.parent_linking_issn = "" self.parent_isbn = "" self.parent_uri = "" self.component_title = "" self.component_authors = "" self.component_publication_date = "" self.component_data_type = "" self.component_doi = "" self.component_proprietary_id = "" self.component_online_issn = "" self.component_print_issn = "" self.component_linking_issn = "" self.component_isbn = "" self.component_uri = "" self.data_type = "" self.section_type = "" self.yop = "" self.access_type = "" self.access_method = "" self.metric_type = "" self.total_count = 0 self.month_counts = {} for month_year_str in get_month_years(begin_date, end_date): self.month_counts[month_year_str] = 0 class RequestData: """This holds the data about a report request :param vendor: The vendor being processed :param target_report_types: The report types to attempt to fetch :param begin_date: The begin date to specify in the request :param end_date: The end date to specify in the request :param save_location: Where the generated report should be saved :param settings: The system's settings object :param special_options: Special options if fetching a special report """ def __init__(self, vendor: Vendor, target_report_types: list, begin_date: QDate, end_date: QDate, save_location: str, settings: SettingsModel, special_options: SpecialReportOptions = None): self.vendor = vendor self.target_report_types = target_report_types self.begin_date = begin_date self.end_date = end_date self.save_location = save_location self.settings = settings self.special_options = special_options class ProcessResult: """This holds the results of an fetch process :param vendor: The target vendor :param report_type: The target report type """ def __init__(self, vendor: Vendor, report_type: str = None): self.vendor = vendor self.report_type = report_type self.completion_status = CompletionStatus.SUCCESSFUL self.message = "" self.retry = False self.file_name = "" self.file_dir = "" self.file_path = "" self.protected_file_path = "" self.year = "" class FetchReportsAbstract: def __init__(self, vendors: list, settings: SettingsModel, widget: QWidget): """This contains common functionality shared between classes that fetch reports :param vendors: The list of vendors in the system :param settings: The system's user settings :param widget: The widget of the tab that this class controls """ # region General self.widget = widget self.vendors = [] self.update_vendors(vendors) self.selected_data = [] # List of ReportData Objects self.retry_data = [] # List of (Vendor, list[report_types])> self.vendor_workers = {} # self.started_processes = 0 self.completed_processes = 0 self.total_processes = 0 self.begin_date = QDate() self.end_date = QDate() self.selected_options = None self.save_dir = "" self.is_cancelling = False self.is_yearly_fetch = False self.settings = settings self.database_report_data = [] # endregion # region Fetch Progress Dialog self.fetch_progress_dialog: QDialog = None self.progress_bar: QProgressBar = None self.status_label: QLabel = None self.scroll_contents: QWidget = None self.scroll_layout: QVBoxLayout = None self.ok_button: QPushButton = None self.retry_button: QPushButton = None self.cancel_button: QPushButton = None self.vendor_result_widgets = {} # # endregion # region Update Database Dialog self.is_updating_database = False self.add_to_database = True self.database_thread = None self.database_worker = None # endregion def on_vendors_changed(self, vendors: list): """Handles the signal emitted when the system's vendor list is updated :param vendors: An updated list of the system's vendors """ self.update_vendors(vendors) self.update_vendors_ui() def update_vendors(self, vendors: list): """ Updates the local copy of vendors that support report fetching (SUSHI) :param vendors: A list of vendors """ self.vendors = [] for vendor in vendors: if vendor.is_non_sushi: continue self.vendors.append(vendor) def update_vendors_ui(self): """Updates the UI to show vendors that support report fetching (SUSHI)""" raise NotImplementedError() def fetch_vendor_data(self, request_data: RequestData): """Initiates the process to fetch reports from a vendor This creates a new thread to work on this vendor :param request_data: The request data for this vendor request """ worker_id = request_data.vendor.name if worker_id in self.vendor_workers: return # Avoid processing a vendor twice vendor_worker = VendorWorker(worker_id, request_data) vendor_worker.worker_finished_signal.connect(self.on_vendor_worker_finished) vendor_thread = QThread() self.vendor_workers[worker_id] = vendor_worker, vendor_thread vendor_worker.moveToThread(vendor_thread) vendor_thread.started.connect(vendor_worker.work) vendor_thread.finished.connect(vendor_thread.deleteLater) vendor_thread.start() if self.settings.show_debug_messages: print(f"{worker_id}: Added a process, total processes: {self.total_processes}") self.update_results_ui(request_data.vendor) def update_results_ui(self, vendor: Vendor, vendor_result: ProcessResult = None, report_results: list = None): """Updates the fetch progress dialog to show results :param vendor: The vendor being updated :param vendor_result: The result of the vendor :param report_results: The results of the vendor's reports """ self.progress_bar.setValue(int((self.completed_processes / self.total_processes) * 100)) if not self.is_cancelling and self.completed_processes != self.total_processes: self.status_label.setText(f"Vendor progress: {self.completed_processes}/{self.total_processes}") if vendor.name in self.vendor_result_widgets: vendor_results_widget, vendor_results_ui = self.vendor_result_widgets[vendor.name] vertical_layout = vendor_results_ui.results_frame.layout() status_label = vendor_results_ui.status_label else: vendor_results_widget = QWidget(self.scroll_contents) vendor_results_ui = VendorResultsWidget.Ui_VendorResultsWidget() vendor_results_ui.setupUi(vendor_results_widget) vendor_results_ui.vendor_label.setText(vendor.name) vertical_layout = vendor_results_ui.results_frame.layout() status_label = vendor_results_ui.status_label frame = vendor_results_ui.results_frame expand_button = vendor_results_ui.expand_button collapse_button = vendor_results_ui.collapse_button status_label.setText("Working...") frame.hide() expand_button.clicked.connect(lambda: frame.show()) collapse_button.clicked.connect(lambda: frame.hide()) self.vendor_result_widgets[vendor.name] = vendor_results_widget, vendor_results_ui self.scroll_layout.addWidget(vendor_results_widget) if vendor_result is None: return status_label.setText("Done") result_widget = self.get_result_widget(vendor, vendor_results_widget, vendor_result) vertical_layout.addWidget(result_widget) for report_result in report_results: result_widget = self.get_result_widget(vendor, vendor_results_widget, report_result) vertical_layout.addWidget(result_widget) def get_result_widget(self, vendor: Vendor, vendor_widget: QWidget, process_result: ProcessResult) -> QWidget: """This creates a result widget for either a vendor or a vendor's report :param vendor: The target vendor :param vendor_widget: The vendor's widget in the fetch progress dialog :param process_result: The result to show """ completion_status = process_result.completion_status report_result_widget = QWidget(vendor_widget) report_result_ui = ReportResultWidget.Ui_ReportResultWidget() report_result_ui.setupUi(report_result_widget) if process_result.message: report_result_ui.message_label.setText(process_result.message) else: report_result_ui.message_label.hide() if process_result.report_type is not None: # If this is a report result, not vendor report_result_ui.report_type_label.setText(process_result.report_type) if completion_status == CompletionStatus.SUCCESSFUL or completion_status == CompletionStatus.WARNING: report_result_ui.file_frame.show() report_result_ui.folder_button.clicked.connect( lambda: GeneralUtils.open_file_or_dir(process_result.file_dir)) report_result_ui.file_label.setText(f"Saved as: {process_result.file_name}") report_result_ui.file_label.mousePressEvent = \ lambda event: GeneralUtils.open_file_or_dir(process_result.file_path) else: report_result_ui.file_frame.hide() else: report_result_ui.report_type_label.setText("Target Reports") report_result_ui.file_frame.hide() report_result_ui.retry_frame.hide() report_result_ui.success_label.setText(process_result.completion_status.value) if completion_status == CompletionStatus.FAILED: report_result_ui.retry_check_box.stateChanged.connect( lambda checked_state: self.on_report_to_retry_toggled(checked_state, vendor, process_result.report_type)) else: report_result_ui.retry_frame.hide() return report_result_widget def on_vendor_worker_finished(self, worker_id: str): """Handles the signal emmited when a vendor worker has finished :param worker_id: The worker ID of the vendor """ self.completed_processes += 1 thread: QThread worker: VendorWorker worker, thread = self.vendor_workers[worker_id] self.update_results_ui(worker.vendor, worker.process_result, worker.report_process_results) if self.is_yearly_fetch: process_result: ProcessResult for process_result in worker.report_process_results: if process_result.completion_status != CompletionStatus.SUCCESSFUL: continue self.database_report_data.append({'file': process_result.protected_file_path, 'vendor': process_result.vendor.name, 'year': process_result.year}) worker.deleteLater() thread.quit() thread.wait() self.vendor_workers.pop(worker_id, None) if self.started_processes < self.total_processes and not self.is_cancelling: request_data = self.selected_data[self.started_processes] self.fetch_vendor_data(request_data) self.started_processes += 1 elif len(self.vendor_workers) == 0: self.finish_fetching_reports() def start_progress_dialog(self, window_title: str): """Sets up and shows the fetch progress dialog :param window_title: The title of the fetch progress dialog """ self.vendor_result_widgets = {} if self.fetch_progress_dialog: self.fetch_progress_dialog.close() self.fetch_progress_dialog = QDialog(self.widget, flags=Qt.Window | Qt.WindowTitleHint | Qt.CustomizeWindowHint) fetch_progress_ui = FetchProgressDialog.Ui_FetchProgressDialog() fetch_progress_ui.setupUi(self.fetch_progress_dialog) self.fetch_progress_dialog.setWindowTitle(window_title) self.progress_bar = fetch_progress_ui.progress_bar self.status_label = fetch_progress_ui.status_label self.scroll_contents = fetch_progress_ui.scroll_area_widget_contents self.scroll_layout = fetch_progress_ui.scroll_area_vertical_layout self.ok_button = fetch_progress_ui.buttonBox.button(QDialogButtonBox.Ok) self.retry_button = fetch_progress_ui.buttonBox.button(QDialogButtonBox.Retry) self.cancel_button = fetch_progress_ui.buttonBox.button(QDialogButtonBox.Cancel) self.ok_button.setEnabled(False) self.retry_button.setEnabled(False) self.retry_button.setText("Retry Selected") self.ok_button.clicked.connect(lambda: self.fetch_progress_dialog.close()) self.retry_button.clicked.connect(lambda: self.retry_selected_reports(window_title)) self.cancel_button.clicked.connect(self.cancel_workers) self.status_label.setText("Starting...") self.fetch_progress_dialog.show() def on_report_to_retry_toggled(self, checked_state: int, vendor: Vendor, report_type): """Handles the signal emmited when a report result's retry checkbox is toggled The report is added the the list of reports to be retried when the 'retry selected reports' button is clicked :param checked_state: The new checked state :param vendor: The vendor that this report belongs to :param report_type: The report type of the report """ if checked_state == Qt.Checked: found = False for i in range(len(self.retry_data)): v, report_types = self.retry_data[i] if v == vendor: report_types.append(report_type) found = True if not found: self.retry_data.append((vendor, [report_type])) elif checked_state == Qt.Unchecked: for i in range(len(self.retry_data)): v, report_types = self.retry_data[i] if v == vendor: report_types.remove(report_type) if len(report_types) == 0: self.retry_data.pop(i) def retry_selected_reports(self, progress_window_title: str): """Retries the selected reports to retry :param progress_window_title: The title of the fetch progress dialog """ if len(self.retry_data) == 0: GeneralUtils.show_message("No report selected") return self.selected_data = [] for vendor, report_types in self.retry_data: request_data = RequestData(vendor, report_types, self.begin_date, self.end_date, self.save_dir, self.settings, self.selected_options) self.selected_data.append(request_data) self.start_progress_dialog(progress_window_title) self.retry_data = [] self.total_processes = len(self.selected_data) self.started_processes = 0 concurrent_vendors = self.settings.concurrent_vendors while self.started_processes < len(self.selected_data) and self.started_processes < concurrent_vendors: request_data = self.selected_data[self.started_processes] self.fetch_vendor_data(request_data) self.started_processes += 1 def finish_fetching_reports(self): """Finishes up the fetch process""" self.started_processes = 0 self.completed_processes = 0 self.total_processes = 0 self.is_cancelling = False self.cancel_button.setEnabled(False) # Start updating database... if self.is_yearly_fetch and len(self.database_report_data) > 0: if not self.start_updating_database(): self.finish_updating_database() else: self.finish_updating_database() def finish_updating_database(self): """Finishes up the database update process""" self.is_updating_database = False self.database_report_data = [] self.ok_button.setEnabled(True) self.retry_button.setEnabled(True) self.status_label.setText("Done!") if self.settings.show_debug_messages: print("Fin!") def cancel_workers(self): """Sends a cancel signal to all vendor workers, updates the UI accordingly""" self.is_cancelling = True self.total_processes = self.started_processes self.status_label.setText(f"Cancelling... (Waiting for started requests to finish)") for worker, thread in self.vendor_workers.values(): worker.set_cancelling() def is_yearly_range(self, begin_date: QDate, end_date: QDate) -> bool: """Checks if a date range will retrieve all available reports for one year :param begin_date: The begin date :param end_date: The end date """ current_date = QDate.currentDate() if begin_date.year() != end_date.year() or begin_date.year() > current_date.year(): return False if begin_date.year() == current_date.year(): if begin_date.month() == 1 and end_date.month() == max(current_date.month() - 1, 1): return True else: if begin_date.month() == 1 and end_date.month() == 12: return True return False def start_updating_database(self) -> bool: """Starts a thread to update the database. Returns True if successfully started""" if self.is_updating_database: if self.settings.show_debug_messages: print("Database is already updating") return False self.is_updating_database = True self.status_label.setText("Updating database...") self.database_thread = QThread() self.database_worker = UpdateDatabaseWorker(self.database_report_data, False) self.database_worker.moveToThread(self.database_thread) def on_progress_changed(progress: int): self.progress_bar.setValue(int((progress / len(self.database_report_data)) * 100)) def on_worker_finished(code): self.database_thread.quit() self.database_thread.wait() self.finish_updating_database() self.database_worker.progress_changed_signal.connect(on_progress_changed) self.database_worker.worker_finished_signal.connect(on_worker_finished) self.database_thread.started.connect(self.database_worker.work) self.database_thread.start() return True class FetchReportsController(FetchReportsAbstract): """Controls the Fetch Reports tab This class fetches master and standard reports with default parameters. The generated TSV files only include the mandatory columns for each report type :param vendors: The list of vendors in the system :param settings: The system's user settings :param widget: The fetch reports widget :param fetch_reports_ui: The UI for the fetch reports widget """ def __init__(self, vendors: list, settings: SettingsModel, widget: QWidget, fetch_reports_ui: FetchReportsTab.Ui_fetch_reports_tab): super().__init__(vendors, settings, widget) # region General current_date = QDate.currentDate() begin_date = QDate(current_date.year(), 1, 1) end_date = QDate(current_date.year(), max(current_date.month() - 1, 1), 1) self.fetch_all_begin_date = QDate(begin_date) self.adv_begin_date = QDate(begin_date) self.fetch_all_end_date = QDate(end_date) self.adv_end_date = QDate(end_date) self.is_last_fetch_advanced = False # endregion # region Start Fetch Buttons self.fetch_all_btn = fetch_reports_ui.fetch_all_data_button self.fetch_all_btn.clicked.connect(self.fetch_all_basic_data) self.fetch_adv_btn = fetch_reports_ui.fetch_advanced_button self.fetch_adv_btn.clicked.connect(self.fetch_advanced_data) # endregion # region Vendors self.vendor_list_view = fetch_reports_ui.vendors_list_view_fetch self.vendor_list_model = QStandardItemModel(self.vendor_list_view) self.vendor_list_view.setModel(self.vendor_list_model) self.update_vendors_ui() self.select_vendors_btn = fetch_reports_ui.select_vendors_button_fetch self.select_vendors_btn.clicked.connect(self.select_all_vendors) self.deselect_vendors_btn = fetch_reports_ui.deselect_vendors_button_fetch self.deselect_vendors_btn.clicked.connect(self.deselect_all_vendors) # endregion # region Report Types self.report_type_list_view = fetch_reports_ui.report_types_list_view self.report_type_list_model = QStandardItemModel(self.report_type_list_view) self.report_type_list_view.setModel(self.report_type_list_model) for report_type in ALL_REPORTS: item = QStandardItem(report_type) item.setCheckable(True) item.setEditable(False) self.report_type_list_model.appendRow(item) self.select_report_types_btn = fetch_reports_ui.select_report_types_button_fetch self.select_report_types_btn.clicked.connect(self.select_all_report_types) self.deselect_report_types_btn = fetch_reports_ui.deselect_report_types_button_fetch self.deselect_report_types_btn.clicked.connect(self.deselect_all_report_types) self.report_types_help_btn = fetch_reports_ui.report_types_help_button self.report_types_help_btn.clicked.connect( lambda: GeneralUtils.show_message("Only reports supported by each respective vendor will be retrieved, " "and unsupported reports will be listed in \"Expand\" results")) # endregion # region Custom Directory self.custom_dir_frame = fetch_reports_ui.custom_dir_frame self.custom_dir_frame.hide() self.custom_dir_message_frame = fetch_reports_ui.frame self.custom_dir_message_frame.hide() self.custom_dir_message_frame2 = fetch_reports_ui.frame_2 self.custom_dir_message_frame2.hide() self.custom_dir_frame_message1 = fetch_reports_ui.label_38 self.custom_dir_frame_message2 = fetch_reports_ui.label self.custom_dir_edit = fetch_reports_ui.custom_dir_edit self.custom_dir_edit.setText(self.settings.other_directory) self.custom_dir_button = fetch_reports_ui.custom_dir_button self.custom_dir_button.clicked.connect(self.on_custom_dir_clicked) self.date_range_help_btn = fetch_reports_ui.date_range_help_button self.date_range_help_btn.clicked.connect( lambda: GeneralUtils.show_message("Reports run for date ranges that represent a normal Jan-Dec calendar" " year, or Jan-last month for calendar years in progress, will be added" " to (or updated in) the search database. All other date ranges will be " "saved in the specified folder but not added to the search database.")) self.date_range_help_btn2 = fetch_reports_ui.date_range_help_button2 self.date_range_help_btn2.clicked.connect( lambda: GeneralUtils.show_message("Reports run for date ranges that represent a normal Jan-Dec calendar" " year, or Jan-last month for calendar years in progress, will be added" " to (or updated in) the search database. All other date ranges will be " "saved in the specified folder but not added to the search database.")) self.date_range_help_btn3 = fetch_reports_ui.date_range_help_button3 self.date_range_help_btn3.clicked.connect( lambda: GeneralUtils.show_message("See Other Reports Directory setting in Settings for default")) # endregion # region Date Edits self.all_date_edit = fetch_reports_ui.All_reports_edit_fetch self.all_date_edit.setDate(self.fetch_all_begin_date) self.all_date_edit.dateChanged.connect(self.on_fetch_all_date_changed) self.begin_date_edit_year = fetch_reports_ui.begin_date_edit_fetch_year self.begin_date_edit_year.setDate(self.adv_begin_date) self.begin_date_edit_year.dateChanged.connect(lambda date: self.on_date_year_changed(date, "adv_begin")) self.end_date_edit_year = fetch_reports_ui.end_date_edit_fetch_year self.end_date_edit_year.setDate(self.adv_end_date) self.end_date_edit_year.dateChanged.connect(lambda date: self.on_date_year_changed(date, "adv_end")) self.begin_month_combo_box = fetch_reports_ui.begin_month_combo_box for month in MONTH_NAMES: self.begin_month_combo_box.addItem(month) self.begin_month_combo_box.currentIndexChanged.connect( lambda index: self.on_date_month_changed(index + 1, "adv_begin")) self.begin_month_combo_box.setCurrentIndex(self.adv_begin_date.month() - 1) self.end_month_combo_box = fetch_reports_ui.end_month_combo_box for month in MONTH_NAMES: self.end_month_combo_box.addItem(month) self.end_month_combo_box.currentIndexChanged.connect( lambda index: self.on_date_month_changed(index + 1, "adv_end")) self.end_month_combo_box.setCurrentIndex(self.adv_end_date.month() - 1) # endregion def update_vendors_ui(self): """Updates the UI to show vendors that support report fetching (SUSHI)""" self.vendor_list_model.clear() for vendor in self.vendors: item = QStandardItem(vendor.name) item.setCheckable(True) item.setEditable(False) self.vendor_list_model.appendRow(item) def on_fetch_all_date_changed(self, date: QDate): """Handles the signal emitted when the 'fetch all' date is changed :param date: The new date """ current_date = QDate.currentDate() if date.year() == current_date.year(): self.fetch_all_begin_date = QDate(current_date.year(), 1, 1) self.fetch_all_end_date = QDate(current_date.year(), max(current_date.month() - 1, 1), 1) elif date.year() < current_date.year(): self.fetch_all_begin_date = QDate(date.year(), 1, 1) self.fetch_all_end_date = QDate(date.year(), 12, 1) else: self.all_date_edit.setDate(current_date) def on_date_year_changed(self, date: QDate, date_type: str): """Handles the signal emitted when a date's year is changed :param date: The new date :param date_type: The date to be updated """ if date_type == "adv_begin": self.adv_begin_date = QDate(date.year(), self.adv_begin_date.month(), self.adv_begin_date.day()) elif date_type == "adv_end": self.adv_end_date = QDate(date.year(), self.adv_end_date.month(), self.adv_end_date.day()) if self.is_yearly_range(self.adv_begin_date, self.adv_end_date): self.custom_dir_frame.hide() self.custom_dir_message_frame.hide() self.custom_dir_message_frame2.hide() else: self.custom_dir_frame.show() if self.custom_dir_frame_message_show(self.adv_begin_date, self.adv_end_date): self.custom_dir_message_frame2.show() self.custom_dir_message_frame.hide() else: self.custom_dir_message_frame.show() self.custom_dir_message_frame2.hide() def on_date_month_changed(self, month: int, date_type: str): """Handles the signal emitted when a date's month is changed :param month: The new month :param date_type: The date to be updated """ if date_type == "adv_begin": self.adv_begin_date = QDate(self.adv_begin_date.year(), month, self.adv_begin_date.day()) elif date_type == "adv_end": self.adv_end_date = QDate(self.adv_end_date.year(), month, self.adv_end_date.day()) if self.is_yearly_range(self.adv_begin_date, self.adv_end_date): self.custom_dir_frame.hide() self.custom_dir_message_frame.hide() self.custom_dir_message_frame2.hide() else: self.custom_dir_frame.show() if self.custom_dir_frame_message_show(self.adv_begin_date, self.adv_end_date): self.custom_dir_message_frame2.show() self.custom_dir_message_frame.hide() else: self.custom_dir_message_frame.show() self.custom_dir_message_frame2.hide() def custom_dir_frame_message_show(self, begin_date: QDate, end_date: QDate) -> bool: """Checks which message will show on the custom dir frame :param begin_date: The begin date :param end_date: The end date """ current_date = QDate.currentDate() if begin_date.year() == end_date.year() == current_date.year(): if begin_date.month() == 1 and end_date.month() == 12: return True return False def on_custom_dir_clicked(self): """Handles the signal emitted when the choose custom directory button is clicked""" dir_path = GeneralUtils.choose_directory() if dir_path: self.custom_dir_edit.setText(dir_path) def select_all_vendors(self): """Checks all vendors in the vendors list view""" for i in range(self.vendor_list_model.rowCount()): self.vendor_list_model.item(i).setCheckState(Qt.Checked) def deselect_all_vendors(self): """Un-checks all vendors in the vendors list view""" for i in range(self.vendor_list_model.rowCount()): self.vendor_list_model.item(i).setCheckState(Qt.Unchecked) def select_all_report_types(self): """Checks all report types in the report types list view""" for i in range(self.report_type_list_model.rowCount()): self.report_type_list_model.item(i).setCheckState(Qt.Checked) def deselect_all_report_types(self): """Un-checks all report types in the report types list view""" for i in range(self.report_type_list_model.rowCount()): self.report_type_list_model.item(i).setCheckState(Qt.Unchecked) def fetch_all_basic_data(self): """Fetches all reports for the selected year""" if self.total_processes > 0: GeneralUtils.show_message(f"Waiting for pending processes to complete...") if self.settings.show_debug_messages: print(f"Waiting for pending processes to complete...") return if len(self.vendors) == 0: GeneralUtils.show_message("Vendor list is empty") return self.begin_date = self.fetch_all_begin_date self.end_date = self.fetch_all_end_date if self.begin_date > self.end_date: GeneralUtils.show_message("\'Begin Date\' is earlier than \'End Date\'") return self.is_yearly_fetch = True self.save_dir = self.settings.yearly_directory self.selected_data = [] for i in range(len(self.vendors)): if self.vendors[i].is_non_sushi: continue request_data = RequestData(self.vendors[i], ALL_REPORTS, self.begin_date, self.end_date, self.save_dir, self.settings) self.selected_data.append(request_data) self.is_last_fetch_advanced = False self.start_progress_dialog("Fetch Reports Progress") self.retry_data = [] self.total_processes = len(self.selected_data) self.started_processes = 0 concurrent_vendors = self.settings.concurrent_vendors while self.started_processes < len(self.selected_data) and self.started_processes < concurrent_vendors: request_data = self.selected_data[self.started_processes] self.fetch_vendor_data(request_data) self.started_processes += 1 def fetch_advanced_data(self): """Fetches reports based on the selected options in the advanced view of the UI""" if self.total_processes > 0: GeneralUtils.show_message(f"Waiting for pending processes to complete...") if self.settings.show_debug_messages: print(f"Waiting for pending processes to complete...") return if len(self.vendors) == 0: GeneralUtils.show_message("Vendor list is empty") return self.begin_date = self.adv_begin_date self.end_date = self.adv_end_date if self.begin_date > self.end_date: GeneralUtils.show_message("\'Begin Date\' is earlier than \'End Date\'") return self.selected_data = [] selected_report_types = [] for i in range(len(ALL_REPORTS)): if self.report_type_list_model.item(i).checkState() == Qt.Checked: selected_report_types.append(ALL_REPORTS[i]) if len(selected_report_types) == 0: GeneralUtils.show_message("No report type selected") return self.is_yearly_fetch = self.is_yearly_range(self.adv_begin_date, self.adv_end_date) custom_dir = self.custom_dir_edit.text() if not custom_dir: custom_dir = self.settings.other_directory self.save_dir = custom_dir if not self.is_yearly_fetch else self.settings.yearly_directory for i in range(self.vendor_list_model.rowCount()): if self.vendor_list_model.item(i).checkState() == Qt.Checked: request_data = RequestData(self.vendors[i], selected_report_types, self.begin_date, self.end_date, self.save_dir, self.settings) self.selected_data.append(request_data) if len(self.selected_data) == 0: GeneralUtils.show_message("No vendor selected") return self.start_progress_dialog("Fetch Reports Progress") self.is_last_fetch_advanced = False self.retry_data = [] self.total_processes = len(self.selected_data) self.started_processes = 0 concurrent_vendors = self.settings.concurrent_vendors while self.started_processes < len(self.selected_data) and self.started_processes < concurrent_vendors: request_data = self.selected_data[self.started_processes] self.fetch_vendor_data(request_data) self.started_processes += 1 class FetchSpecialReportsController(FetchReportsAbstract): def __init__(self, vendors: list, settings: SettingsModel, widget: QWidget, fetch_special_reports_ui: FetchSpecialReportsTab.Ui_fetch_special_reports_tab): super().__init__(vendors, settings, widget) # region General self.selected_report_type = None self.selected_options = SpecialReportOptions() current_date = QDate.currentDate() self.begin_date = QDate(current_date.year(), 1, 1) self.end_date = QDate(current_date.year(), max(current_date.month() - 1, 1), 1) # endregion # region Start Fetch Button self.fetch_special_btn = fetch_special_reports_ui.fetch_special_data_button self.fetch_special_btn.clicked.connect(self.fetch_special_data) # endregion # region Vendors self.vendor_list_view = fetch_special_reports_ui.vendors_list_view_special self.vendor_list_model = QStandardItemModel(self.vendor_list_view) self.vendor_list_view.setModel(self.vendor_list_model) self.update_vendors_ui() self.select_vendors_btn = fetch_special_reports_ui.select_vendors_button_special self.select_vendors_btn.clicked.connect(self.select_all_vendors) self.deselect_vendors_btn = fetch_special_reports_ui.deselect_vendors_button_special self.deselect_vendors_btn.clicked.connect(self.deselect_all_vendors) # endregion # region Options self.options_frame = fetch_special_reports_ui.options_frame self.options_layout = self.options_frame.layout() help_message = "The 'show' checkboxes will break down your data by displaying extra columns with the " \ "selected attributes (usually results in more lines per item)\n" \ "Filters will limit your data to just the selected options (implies also showing that " \ "attribute).\n" \ "You can show the attribute breakdown without selecting filtering to specific values." fetch_special_reports_ui.options_help_button.clicked.connect( lambda: GeneralUtils.show_message(help_message) ) # endregion # region Report Types self.pr_radio_button = fetch_special_reports_ui.pr_radio_button self.dr_radio_button = fetch_special_reports_ui.dr_radio_button self.tr_radio_button = fetch_special_reports_ui.tr_radio_button self.ir_radio_button = fetch_special_reports_ui.ir_radio_button self.pr_radio_button.clicked.connect(lambda checked: self.on_report_type_selected(MajorReportType.PLATFORM)) self.dr_radio_button.clicked.connect(lambda checked: self.on_report_type_selected(MajorReportType.DATABASE)) self.tr_radio_button.clicked.connect(lambda checked: self.on_report_type_selected(MajorReportType.TITLE)) self.ir_radio_button.clicked.connect(lambda checked: self.on_report_type_selected(MajorReportType.ITEM)) self.pr_radio_button.setChecked(True) self.on_report_type_selected(MajorReportType.PLATFORM) # endregion # region Date Edits self.date_range_help_btn = fetch_special_reports_ui.date_range_help_button self.date_range_help_btn.clicked.connect( lambda: GeneralUtils.show_message("Many vendors do not support a date range longer than 12 consecutive months.")) self.begin_date_edit_year = fetch_special_reports_ui.begin_date_edit_special_year self.begin_date_edit_year.setDate(self.begin_date) self.begin_date_edit_year.dateChanged.connect(lambda date: self.on_date_year_changed(date, "begin_date")) self.end_date_edit_year = fetch_special_reports_ui.end_date_edit_special_year self.end_date_edit_year.setDate(self.end_date) self.end_date_edit_year.dateChanged.connect(lambda date: self.on_date_year_changed(date, "end_date")) self.begin_month_combo_box = fetch_special_reports_ui.begin_month_combo_box for month in MONTH_NAMES: self.begin_month_combo_box.addItem(month) self.begin_month_combo_box.currentIndexChanged.connect( lambda index: self.on_date_month_changed(index + 1, "begin_date")) self.begin_month_combo_box.setCurrentIndex(self.begin_date.month() - 1) self.end_month_combo_box = fetch_special_reports_ui.end_month_combo_box for month in MONTH_NAMES: self.end_month_combo_box.addItem(month) self.end_month_combo_box.currentIndexChanged.connect( lambda index: self.on_date_month_changed(index + 1, "end_date")) self.end_month_combo_box.setCurrentIndex(self.end_date.month() - 1) # endregion # region Custom Directory self.custom_dir_frame = fetch_special_reports_ui.custom_dir_frame self.custom_dir_edit = fetch_special_reports_ui.custom_dir_edit self.custom_dir_edit.setText(self.settings.other_directory) self.custom_dir_button = fetch_special_reports_ui.custom_dir_button self.custom_dir_button.clicked.connect(self.on_custom_dir_clicked) self.custom_dir_help_btn = fetch_special_reports_ui.custom_dir_help_button self.custom_dir_help_btn.clicked.connect( lambda: GeneralUtils.show_message("See Other Reports Directory setting in Settings for default.")) # endregion def update_vendors_ui(self): """Updates the UI to show vendors that support report fetching (SUSHI)""" self.vendor_list_model.clear() for vendor in self.vendors: item = QStandardItem(vendor.name) item.setCheckable(True) item.setEditable(False) self.vendor_list_model.appendRow(item) def on_date_year_changed(self, date: QDate, date_type: str): """Handles the signal emitted when a date's year is changed :param date: The new date :param date_type: The date to be updated """ if date_type == "begin_date": self.begin_date = QDate(date.year(),self.begin_date.month(),self.begin_date.day()) elif date_type == "end_date": self.end_date = QDate(date.year(),self.end_date.month(),self.end_date.day()) def on_date_month_changed(self, month: int, date_type: str): """Handles the signal emitted when a date's month is changed :param month: The new month :param date_type: The date to be updated """ if date_type == "begin_date": self.begin_date = QDate(self.begin_date.year(), month, self.begin_date.day()) elif date_type == "end_date": self.end_date = QDate(self.end_date.year(), month, self.end_date.day()) def on_report_type_selected(self, major_report_type: MajorReportType): """Handles the signal emitted when a report type is selected :param major_report_type: The major report type (Only master reports are supported) """ if major_report_type == self.selected_report_type: return self.selected_report_type = major_report_type self.selected_options = SpecialReportOptions() # Remove existing options from ui for i in reversed(range(self.options_layout.count())): widget = self.options_layout.itemAt(i).widget() # remove it from the layout list self.options_layout.removeWidget(widget) # remove it from the gui widget.deleteLater() # Add new options self.options_layout.addWidget(QLabel("Show", self.options_frame), 0, 0) self.options_layout.addWidget(QLabel("Filters", self.options_frame), 0, 1) special_options = SPECIAL_REPORT_OPTIONS[major_report_type] for i in range(len(special_options)): option_name = special_options[i][1] checkbox = QCheckBox(option_name, self.options_frame) checkbox.toggled.connect( lambda is_checked, option=option_name: self.on_special_option_toggled(option, is_checked)) self.options_layout.addWidget(checkbox, i + 1, 0) option_type: SpecialOptionType = special_options[i][0] if option_type == SpecialOptionType.AP or option_type == SpecialOptionType.POS\ or option_type == SpecialOptionType.ADP: line_edit = QLineEdit(DEFAULT_SPECIAL_OPTION_VALUE, self.options_frame) line_edit.setReadOnly(True) button = QPushButton("Choose", self.options_frame) if option_type == SpecialOptionType.ADP: button.clicked.connect( lambda c, so=special_options[i], edit=line_edit: self.on_special_date_parameter_option_button_clicked(so, edit)) else: button.clicked.connect( lambda c, so=special_options[i], edit=line_edit: self.on_special_parameter_option_button_clicked(so, edit)) self.options_layout.addWidget(line_edit, i + 1, 1) self.options_layout.addWidget(button, i + 1, 2) def on_special_option_toggled(self, option: str, is_checked: bool): """Handles the signal emitted when a special option is checked or un-checked :param option: The special option :param is_checked: Checked or un-checked """ option = option.lower() __, option_type, option_name, curr_options = self.selected_options.__getattribute__(option) self.selected_options.__setattr__(option, (is_checked, option_type, option_name, curr_options)) def on_special_parameter_option_button_clicked(self, special_option, line_edit): """Handles the signal emitted when a special option with parameters clicked :param special_option: The special option :param line_edit: The line edit to show the selected parameters for this option """ option_type, option_name, option_list = special_option is_selected, option_type, option_name, curr_options = self.selected_options.__getattribute__(option_name.lower()) dialog = QDialog(self.options_frame, flags=Qt.Window | Qt.WindowTitleHint | Qt.CustomizeWindowHint) dialog.setWindowTitle(option_name + " options") layout = QVBoxLayout(dialog) list_view = QListView(dialog) list_view.setAlternatingRowColors(True) model = QStandardItemModel(list_view) for option in option_list: item = QStandardItem(option) item.setCheckable(True) item.setEditable(False) if option in curr_options: item.setCheckState(Qt.Checked) model.appendRow(item) list_view.setModel(model) layout.addWidget(list_view) def on_ok_button_clicked(): checked_list = [] for i in range(model.rowCount()): if model.item(i).checkState() == Qt.Checked: checked_list.append(model.item(i).text()) if len(checked_list) == 0: checked_list = [DEFAULT_SPECIAL_OPTION_VALUE] line_edit.setText("|".join(checked_list)) self.selected_options.__setattr__(option_name.lower(), (is_selected, option_type, option_name, checked_list)) dialog.close() button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel, dialog) button_box.accepted.connect(on_ok_button_clicked) button_box.rejected.connect(lambda: dialog.close()) button_box.setCenterButtons(True) layout.addWidget(button_box) dialog.exec_() def on_special_date_parameter_option_button_clicked(self, special_option, line_edit): """Handles the signal emitted when a date special option is clicked :param special_option: The special option :param line_edit: The line edit to show the selected date parameters for this option """ option_type, option_name = special_option is_selected, option_type, option_name, selected_options = self.selected_options.__getattribute__(option_name.lower()) dialog = QDialog(self.options_frame, flags=Qt.Window | Qt.WindowTitleHint | Qt.CustomizeWindowHint) dialog.setWindowTitle(option_name + " options") layout = QVBoxLayout(dialog) radio_button_group = QButtonGroup(dialog) default_frame = QFrame(dialog) default_frame.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) default_layout = QHBoxLayout(default_frame) default_layout.setContentsMargins(0, 0, 0, 0) default_radio_btn = QRadioButton(default_frame) default_radio_btn.setChecked(True) radio_button_group.addButton(default_radio_btn) default_label = QLabel(DEFAULT_SPECIAL_OPTION_VALUE, dialog) default_layout.addWidget(default_radio_btn) default_layout.addWidget(default_label) single_date_frame = QFrame(dialog) single_date_frame.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) single_date_layout = QHBoxLayout(single_date_frame) single_date_layout.setContentsMargins(0, 0, 0, 0) single_date_radio_btn = QRadioButton(single_date_frame) radio_button_group.addButton(single_date_radio_btn) single_date_edit = QDateEdit(QDate.currentDate(), single_date_frame) single_date_edit.setDisplayFormat("yyyy") single_date_layout.addWidget(single_date_radio_btn) single_date_layout.addWidget(single_date_edit) range_date_frame = QFrame(dialog) range_date_frame.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) range_date_layout = QHBoxLayout(range_date_frame) range_date_layout.setContentsMargins(0, 0, 0, 0) range_date_radio_btn = QRadioButton(range_date_frame) radio_button_group.addButton(range_date_radio_btn) begin_date_edit = QDateEdit(QDate.currentDate(), range_date_frame) end_date_edit = QDateEdit(QDate.currentDate(), range_date_frame) begin_date_edit.setDisplayFormat("yyyy") end_date_edit.setDisplayFormat("yyyy") to_label = QLabel(" to ", range_date_frame) to_label.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) range_date_layout.addWidget(range_date_radio_btn) range_date_layout.addWidget(begin_date_edit) range_date_layout.addWidget(to_label) range_date_layout.addWidget(end_date_edit) def on_ok_button_clicked(): new_selection = [DEFAULT_SPECIAL_OPTION_VALUE] checked_button = radio_button_group.checkedButton() if checked_button == single_date_radio_btn: new_selection = [single_date_edit.date().toString("yyyy")] elif checked_button == range_date_radio_btn: new_selection = [begin_date_edit.date().toString("yyyy") + "-" + end_date_edit.date().toString("yyyy")] line_edit.setText(new_selection[0]) self.selected_options.__setattr__(option_name.lower(), (is_selected, option_type, option_name, new_selection)) dialog.close() button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel, dialog) button_box.accepted.connect(on_ok_button_clicked) button_box.rejected.connect(lambda: dialog.close()) button_box.setCenterButtons(True) layout.addWidget(default_frame) layout.addWidget(single_date_frame) layout.addWidget(range_date_frame) layout.addWidget(button_box) dialog.exec_() def on_custom_dir_clicked(self): """Handles the signal emitted when the choose custom directory button is clicked""" dir_path = GeneralUtils.choose_directory() if dir_path: self.custom_dir_edit.setText(dir_path) def select_all_vendors(self): """Checks all vendors in the vendors list view""" for i in range(self.vendor_list_model.rowCount()): self.vendor_list_model.item(i).setCheckState(Qt.Checked) def deselect_all_vendors(self): """Un-checks all vendors in the vendors list view""" for i in range(self.vendor_list_model.rowCount()): self.vendor_list_model.item(i).setCheckState(Qt.Unchecked) def fetch_special_data(self): """Fetches reports based on the selected options in the UI""" if self.total_processes > 0: GeneralUtils.show_message(f"Waiting for pending processes to complete...") if self.settings.show_debug_messages: print(f"Waiting for pending processes to complete...") return if len(self.vendors) == 0: GeneralUtils.show_message("Vendor list is empty") return if self.begin_date > self.end_date: GeneralUtils.show_message("\'Begin Date\' is earlier than \'End Date\'") return self.selected_data = [] selected_report_types = [self.selected_report_type.value] self.is_yearly_fetch = False custom_dir = self.custom_dir_edit.text() self.save_dir = custom_dir if custom_dir else self.settings.other_directory for i in range(self.vendor_list_model.rowCount()): if self.vendor_list_model.item(i).checkState() == Qt.Checked: request_data = RequestData(self.vendors[i], selected_report_types, self.begin_date, self.end_date, self.save_dir, self.settings, self.selected_options) self.selected_data.append(request_data) if len(self.selected_data) == 0: GeneralUtils.show_message("No vendor selected") return self.start_progress_dialog("Fetch Special Reports Progress") self.retry_data = [] self.total_processes = len(self.selected_data) self.started_processes = 0 concurrent_vendors = self.settings.concurrent_vendors while self.started_processes < len(self.selected_data) and self.started_processes < concurrent_vendors: request_data = self.selected_data[self.started_processes] self.fetch_vendor_data(request_data) self.started_processes += 1 class VendorWorker(QObject): """This does all the work for a vendor when fetching reports :param worker_id: The ID to identify this worker (vendor_name) :param request_data: The request data for this request """ worker_finished_signal = pyqtSignal(str) def __init__(self, worker_id: str, request_data: RequestData): super().__init__() self.worker_id = worker_id self.request_data = request_data self.vendor = request_data.vendor self.target_report_types = request_data.target_report_types self.show_debug = request_data.settings.show_debug_messages self.concurrent_reports = request_data.settings.concurrent_reports self.request_interval = request_data.settings.request_interval self.request_timeout = request_data.settings.request_timeout self.user_agent = request_data.settings.user_agent self.reports_to_process = [] self.started_processes = 0 self.completed_processes = 0 self.total_processes = 0 self.process_result = ProcessResult(self.vendor) self.report_process_results = [] self.report_workers = {} # self.is_cancelling = False def work(self): """Processes the vendor's requests Requests the vendor's supported reports before requesting only the supported reports """ if self.show_debug: print(f"{self.vendor.name}: Fetching supported reports") request_query = {} if self.vendor.customer_id.strip(): request_query["customer_id"] = self.vendor.customer_id if self.vendor.requestor_id.strip(): request_query["requestor_id"] = self.vendor.requestor_id if self.vendor.api_key.strip(): request_query["api_key"] = self.vendor.api_key if self.vendor.platform.strip(): request_query["platform"] = self.vendor.platform request_url = self.vendor.base_url try: # Some vendors only work if they think a web browser is making the request... response = requests.get(request_url, request_query, headers={'User-Agent': self.user_agent}, timeout=self.request_timeout) if self.show_debug: print(response.url) if response.status_code == 200: self.process_response(response) else: self.process_result.completion_status = CompletionStatus.FAILED self.process_result.message = f"Unexpected HTTP status code received: {response.status_code}" except requests.exceptions.Timeout as e: self.process_result.completion_status = CompletionStatus.FAILED self.process_result.message = f"Request timed out after {self.request_timeout} second(s)" if self.show_debug: print(f"{self.vendor.name}: Request timed out") except requests.exceptions.RequestException as e: self.process_result.completion_status = CompletionStatus.FAILED self.process_result.message = f"Request Exception: {e}" if self.show_debug: print(f"{self.vendor.name}: Request Exception: {e}") if len(self.report_workers) == 0: self.notify_worker_finished() def process_response(self, response: requests.Response): """Processes the response from a REST request Requests the target reports that are supported by the vendor """ if self.is_cancelling: self.process_result.message = "Target reports not processed" self.process_result.completion_status = CompletionStatus.CANCELLED return try: json_response = response.json() exceptions = self.check_for_exception(json_response) if len(exceptions) > 0: self.process_result.message = exception_models_to_message(exceptions) self.process_result.completion_status = CompletionStatus.FAILED return json_dicts = [] if type(json_response) is dict: # This should never be a dict by the standard, but some vendors...... json_dicts = json_response["Report_Items"] if "Report_Items" in json_response else [] # Report_Items??? # Some other vendor might implement it in a different way.......................... elif type(json_response) is list: json_dicts = json_response if len(json_dicts) == 0: raise Exception("JSON is empty") supported_report_types = [] self.reports_to_process = [] for json_dict in json_dicts: supported_report = SupportedReportModel.from_json(json_dict) supported_report_types.append(supported_report.report_id) if supported_report.report_id in self.target_report_types: self.reports_to_process.append(supported_report.report_id) unsupported_report_types = list(set(self.target_report_types) - set(supported_report_types)) self.process_result.message = "Supported by vendor: " self.process_result.message += ", ".join(self.reports_to_process) self.process_result.message += "\nUnsupported: " self.process_result.message += ", ".join(unsupported_report_types) if len(self.reports_to_process) == 0: return self.total_processes = len(self.reports_to_process) self.started_processes = 0 while self.started_processes < self.total_processes and self.started_processes < self.concurrent_reports: QThread.currentThread().sleep(self.request_interval) # Avoid spamming vendor's server if self.is_cancelling: self.process_result.completion_status = CompletionStatus.CANCELLED return self.fetch_report(self.reports_to_process[self.started_processes]) self.started_processes += 1 except json.JSONDecodeError as e: self.process_result.completion_status = CompletionStatus.FAILED self.process_result.message = f"JSON Exception: {e}" if self.show_debug: print(f"{self.vendor.name}: JSON Exception: {e.msg}") except Exception as e: self.process_result.completion_status = CompletionStatus.FAILED self.process_result.message = str(e) if self.show_debug: print(f"{self.vendor.name}: Exception: {e}") def fetch_report(self, report_type: str): """Initiates the process to fetch a report :param report_type: The target report type """ worker_id = report_type if worker_id in self.report_workers: return # Avoid fetching a report twice, app will crash!! report_worker = ReportWorker(worker_id, report_type, self.request_data) report_worker.worker_finished_signal.connect(self.on_report_worker_finished) report_thread = QThread() self.report_workers[worker_id] = report_worker, report_thread report_worker.moveToThread(report_thread) report_thread.started.connect(report_worker.work) report_thread.finished.connect(report_thread.deleteLater) report_thread.start() def check_for_exception(self, json_response) -> list: """Checks a JSON response for exception models :param json_response: The JSON response to be processed """ exceptions = [] if type(json_response) is dict: if "Exception" in json_response: exceptions.append(ExceptionModel.from_json(json_response["Exception"])) # raise Exception(f"Code: {exception.code}, Message: {exception.message}") code = int(json_response["Code"]) if "Code" in json_response else "" message = json_response["Message"] if "Message" in json_response else "" data = json_response["Data"] if "Data" in json_response else "" severity = json_response["Severity"] if "Severity" in json_response else "" if code: exceptions.append(ExceptionModel(code, message, severity, data)) elif type(json_response) is list: for json_dict in json_response: exception = ExceptionModel.from_json(json_dict) if exception.code: exceptions.append(exception) return exceptions def notify_worker_finished(self): """Notifies any listeners that this worker has finished""" self.worker_finished_signal.emit(self.vendor.name) def on_report_worker_finished(self, worker_id: str): """Handles the signal emmited when a report worker has finished :param worker_id: The report worker's worker id """ self.completed_processes += 1 thread: QThread worker: ReportWorker worker, thread = self.report_workers[worker_id] self.report_process_results.append(worker.process_result) worker.deleteLater() thread.quit() thread.wait() self.report_workers.pop(worker_id, None) if self.started_processes < self.total_processes and not self.is_cancelling: QThread.currentThread().sleep(self.request_interval) # Avoid spamming vendor's server self.fetch_report(self.reports_to_process[self.started_processes]) self.started_processes += 1 if len(self.report_workers) == 0: self.notify_worker_finished() def set_cancelling(self): """Sets the worker to a cancelling state""" self.is_cancelling = True class ReportWorker(QObject): """This does all the work for a report :param worker_id: The ID to identify this worker (vendor_name-report_type) :param report_type: The report type to be processed :param request_data: The request data for this request """ worker_finished_signal = pyqtSignal(str) def __init__(self, worker_id: str, report_type: str, request_data: RequestData): super().__init__() self.worker_id = worker_id self.report_type = report_type self.vendor = request_data.vendor self.begin_date = request_data.begin_date self.end_date = request_data.end_date self.show_debug = request_data.settings.show_debug_messages self.request_timeout = request_data.settings.request_timeout self.user_agent = request_data.settings.user_agent self.save_dir = request_data.save_location self.special_options = request_data.special_options self.is_yearly = self.save_dir == request_data.settings.yearly_directory self.is_special = self.special_options is not None self.is_master = self.report_type in MASTER_REPORTS self.process_result = ProcessResult(self.vendor, self.report_type) self.retried_request = False def work(self): """Processes the report request""" if self.show_debug: print(f"{self.vendor.name}-{self.report_type}: Fetching Report") self.make_request() if self.show_debug: print(f"{self.vendor.name}-{self.report_type}: Done") self.notify_worker_finished() def make_request(self): """Sends the request fetch a report""" request_query = {} if self.vendor.customer_id.strip(): request_query["customer_id"] = self.vendor.customer_id if self.vendor.requestor_id.strip(): request_query["requestor_id"] = self.vendor.requestor_id if self.vendor.api_key.strip(): request_query["api_key"] = self.vendor.api_key if self.vendor.platform.strip(): request_query["platform"] = self.vendor.platform request_query["begin_date"] = self.begin_date.toString("yyyy-MM") request_query["end_date"] = self.end_date.toString("yyyy-MM") attributes_to_show = "" if self.is_special: attr_count = 0 special_options_dict = self.special_options.__dict__ for option in special_options_dict: value = special_options_dict[option] is_selected, option_type, option_name, option_parameters = value if is_selected: if option_type == SpecialOptionType.AP or option_type == SpecialOptionType.ADP\ or option_type == SpecialOptionType.POS: if option_parameters[0] != DEFAULT_SPECIAL_OPTION_VALUE: request_query[option] = "|".join(option_parameters) elif option_type == SpecialOptionType.POB: request_query[option] = "True" if option_type == SpecialOptionType.AP or option_type == SpecialOptionType.ADP\ or option_type == SpecialOptionType.AO: if attr_count == 0: attributes_to_show += option_name attr_count += 1 elif attr_count > 0: attributes_to_show += f"|{option_name}" attr_count += 1 elif self.is_yearly and self.is_master: major_report_type = GeneralUtils.get_major_report_type(self.report_type) if major_report_type == MajorReportType.PLATFORM: attributes_to_show = "|".join(PLATFORM_REPORTS_ATTRIBUTES) elif major_report_type == MajorReportType.DATABASE: attributes_to_show = "|".join(DATABASE_REPORTS_ATTRIBUTES) elif major_report_type == MajorReportType.TITLE: attributes_to_show = "|".join(TITLE_REPORTS_ATTRIBUTES) elif major_report_type == MajorReportType.ITEM: attributes_to_show = "|".join(ITEM_REPORTS_ATTRIBUTES) request_query["include_parent_details"] = True request_query["include_component_details"] = True if attributes_to_show: request_query["attributes_to_show"] = attributes_to_show request_url = f"{self.vendor.base_url}/{self.report_type.lower()}" try: # Some vendors only work if they think a web browser is making the request... response = requests.get(request_url, request_query, headers={'User-Agent': self.user_agent}, timeout=self.request_timeout) if self.show_debug: print(response.url) if response.status_code == 200: self.process_response(response) else: self.process_result.completion_status = CompletionStatus.FAILED self.process_result.message = f"Unexpected HTTP status code received: {response.status_code}" except requests.exceptions.Timeout as e: self.process_result.completion_status = CompletionStatus.FAILED self.process_result.message = f"Request timed out after {self.request_timeout} second(s)" if self.show_debug: print(f"{self.vendor.name}: Request timed out") except requests.exceptions.RequestException as e: self.process_result.completion_status = CompletionStatus.FAILED self.process_result.message = f"Request Exception: {e}" if self.show_debug: print( f"{self.vendor.name}-{self.report_type}: Request Exception: {e}") def process_response(self, response: requests.Response): """Processes the response from a request Converts the receoved JSON to a SUSHI Report Model :param response: The received response """ try: json_string = response.text if self.is_yearly: self.save_json_file(json_string) json_dict = json.loads(json_string) report_model = ReportModel.from_json(json_dict) self.process_report_model(report_model) if len(report_model.report_items) > 0 or len(report_model.exceptions) > 0: self.process_result.message = exception_models_to_message(report_model.exceptions) else: self.process_result.message = "Report items not received. No exception received." except json.JSONDecodeError as e: self.process_result.completion_status = CompletionStatus.FAILED if e.msg == "Expecting value": self.process_result.message = f"Vendor did not return any data" else: self.process_result.message = f"JSON Exception: {e.msg}" if self.show_debug: print( f"{self.vendor.name}-{self.report_type}: JSON Exception: {e.msg}") except RetryLaterException as e: if not self.retried_request: if self.show_debug: print(f"{self.vendor.name}-{self.report_type}: Retry Later Exception: {e}") print(f"{self.vendor.name}-{self.report_type}: Retrying in {RETRY_WAIT_TIME} seconds...") QThread.currentThread().sleep(RETRY_WAIT_TIME) # Wait some time before retrying request self.retried_request = True self.make_request() else: self.process_result.message = "Retry later exception received" message = exception_models_to_message(e.exceptions) if message: self.process_result.message += "\n\n" + message self.process_result.completion_status = CompletionStatus.FAILED self.process_result.retry = True if self.show_debug: print( f"{self.vendor.name}-{self.report_type}: Retry Later Exception: {e}") except ReportHeaderMissingException as e: self.process_result.message = "Report_Header not received, no file was created" message = exception_models_to_message(e.exceptions) if message: self.process_result.message += "\n\n" + message self.process_result.completion_status = CompletionStatus.FAILED if self.show_debug: print( f"{self.vendor.name}-{self.report_type}: Report Header Missing Exception: {e}") except UnacceptableCodeException as e: self.process_result.message = "Unsupported exception code received" message = exception_models_to_message(e.exceptions) if message: self.process_result.message += "\n\n" + message self.process_result.completion_status = CompletionStatus.FAILED if self.show_debug: print( f"{self.vendor.name}-{self.report_type}: Unsupported Code Exception: {e}") except Exception as e: self.process_result.completion_status = CompletionStatus.FAILED self.process_result.message = str(e) if self.show_debug: print(f"{self.vendor.name}-{self.report_type}: Exception: {e}") def process_report_model(self, report_model: ReportModel): """Processes the report model into a TSV report :param report_model: The report model """ report_type = report_model.report_header.report_id major_report_type = report_model.report_header.major_report_type report_items = report_model.report_items report_rows = [] file_dir = f"{self.save_dir}{self.begin_date.toString('yyyy')}/{self.vendor.name}/" file_name = f"{self.begin_date.toString('yyyy')}_{self.vendor.name}_{report_type}.tsv" file_path = f"{file_dir}{file_name}" if self.show_debug: print(f"{self.vendor.name}-{self.report_type}: Processing report") for report_item in report_items: metric_row_dict = {} # Some metric_types have a list of components # Some Item report metric_types have a list of components components = [] # list({component_values_as_dict}) performance: PerformanceModel for performance in report_item.performances: begin_month = QDate.fromString(performance.period.begin_date, "yyyy-MM-dd").toString("MMM-yyyy") instance: InstanceModel for instance in performance.instances: metric_type = instance.metric_type if metric_type not in metric_row_dict: metric_row = ReportRow(self.begin_date, self.end_date) metric_row.metric_type = metric_type metric_row_dict[metric_type] = metric_row else: metric_row = metric_row_dict[metric_type] if major_report_type == MajorReportType.PLATFORM: report_item: PlatformReportItemModel if report_item.platform: metric_row.platform = report_item.platform if report_item.data_type: metric_row.data_type = report_item.data_type if report_item.access_method: metric_row.access_method = report_item.access_method elif major_report_type == MajorReportType.DATABASE: report_item: DatabaseReportItemModel if report_item.database: metric_row.database = report_item.database if report_item.publisher: metric_row.publisher = report_item.publisher if report_item.platform: metric_row.platform = report_item.platform if report_item.data_type: metric_row.data_type = report_item.data_type if report_item.access_method: metric_row.access_method = report_item.access_method pub_id_str = "" for pub_id in report_item.publisher_ids: pub_id_str += f"{pub_id.item_type}:{pub_id.value}; " if pub_id_str: metric_row.publisher_id = pub_id_str for item_id in report_item.item_ids: if item_id.item_type == "Proprietary" or item_id.item_type == "Proprietary_ID": metric_row.proprietary_id = item_id.value elif major_report_type == MajorReportType.TITLE: report_item: TitleReportItemModel if report_item.title: metric_row.title = report_item.title if report_item.publisher: metric_row.publisher = report_item.publisher if report_item.platform: metric_row.platform = report_item.platform if report_item.data_type: metric_row.data_type = report_item.data_type if report_item.section_type: metric_row.section_type = report_item.section_type if report_item.yop: metric_row.yop = report_item.yop if report_item.access_type: metric_row.access_type = report_item.access_type if report_item.access_method: metric_row.access_method = report_item.access_method pub_id_str = "" for pub_id in report_item.publisher_ids: pub_id_str += f"{pub_id.item_type}:{pub_id.value}; " if pub_id_str: metric_row.publisher_id = pub_id_str item_id: TypeValueModel for item_id in report_item.item_ids: item_type = item_id.item_type if item_type == "DOI": metric_row.doi = item_id.value elif item_type == "Proprietary" or item_type == "Proprietary_ID": metric_row.proprietary_id = item_id.value elif item_type == "ISBN": metric_row.isbn = item_id.value elif item_type == "Print_ISSN": metric_row.print_issn = item_id.value elif item_type == "Online_ISSN": metric_row.online_issn = item_id.value elif item_type == "Linking_ISSN": metric_row.linking_issn = item_id.value elif item_type == "URI": metric_row.uri = item_id.value elif major_report_type == MajorReportType.ITEM: report_item: ItemReportItemModel if report_item.item: metric_row.item = report_item.item if report_item.publisher: metric_row.publisher = report_item.publisher if report_item.platform: metric_row.platform = report_item.platform if report_item.data_type: metric_row.data_type = report_item.data_type if report_item.yop: metric_row.yop = report_item.yop if report_item.access_type: metric_row.access_type = report_item.access_type if report_item.access_method: metric_row.access_method = report_item.access_method # Publisher ID pub_id_str = "" for pub_id in report_item.publisher_ids: pub_id_str += f"{pub_id.item_type}:{pub_id.value}; " if pub_id_str: metric_row.publisher_id = pub_id_str # Authors authors_str = "" item_contributor: ItemContributorModel for item_contributor in report_item.item_contributors: if item_contributor.item_type == "Author": authors_str += f"{item_contributor.name}" if item_contributor.identifier: authors_str += f" ({item_contributor.identifier})" authors_str += "; " if authors_str: metric_row.authors = authors_str.rstrip("; ") # Publication date item_date: TypeValueModel for item_date in report_item.item_dates: if item_date.item_type == "Publication_Date": metric_row.publication_date = item_date.value # Article version item_attribute: TypeValueModel for item_attribute in report_item.item_attributes: if item_attribute.item_type == "Article_Version": metric_row.article_version = item_attribute.value # Base IDs item_id: TypeValueModel for item_id in report_item.item_ids: item_type = item_id.item_type if item_type == "DOI": metric_row.doi = item_id.value elif item_type == "Proprietary" or item_type == "Proprietary_ID": metric_row.proprietary_id = item_id.value elif item_type == "ISBN": metric_row.isbn = item_id.value elif item_type == "Print_ISSN": metric_row.print_issn = item_id.value elif item_type == "Online_ISSN": metric_row.online_issn = item_id.value elif item_type == "Linking_ISSN": metric_row.linking_issn = item_id.value elif item_type == "URI": metric_row.uri = item_id.value # Parent if report_item.item_parent is not None: item_parent: ItemParentModel item_parent = report_item.item_parent if item_parent.item_name: metric_row.parent_title = item_parent.item_name if item_parent.data_type: metric_row.parent_data_type = item_parent.data_type # Authors authors_str = "" item_contributor: ItemContributorModel for item_contributor in report_item.item_contributors: if item_contributor.item_type == "Author": authors_str += f"{item_contributor.name}" if item_contributor.identifier: authors_str += f" ({item_contributor.identifier})" authors_str += "; " authors_str.rstrip("; ") if authors_str: metric_row.authors = authors_str # Publication date item_date: TypeValueModel for item_date in item_parent.item_dates: if item_date.item_type == "Publication_Date" or item_date.item_type == "Pub_Date": metric_row.parent_publication_date = item_date.value # Article version item_attribute: TypeValueModel for item_attribute in item_parent.item_attributes: if item_attribute.item_type == "Article_Version": metric_row.parent_article_version = item_attribute.value # Parent IDs item_id: TypeValueModel for item_id in item_parent.item_ids: item_type = item_id.item_type if item_type == "DOI": metric_row.parent_doi = item_id.value elif item_type == "Proprietary" or item_type == "Proprietary_ID": metric_row.parent_proprietary_id = item_id.value elif item_type == "ISBN": metric_row.parent_isbn = item_id.value elif item_type == "Print_ISSN": metric_row.parent_print_issn = item_id.value elif item_type == "Online_ISSN": metric_row.parent_online_issn = item_id.value elif item_type == "URI": metric_row.parent_uri = item_id.value else: if self.show_debug: print( f"{self.vendor.name}-{self.report_type}: Unexpected report type") month_counts = metric_row.month_counts month_counts[begin_month] += instance.count metric_row.total_count += instance.count if major_report_type == MajorReportType.ITEM: # Item Components item_component: ItemComponentModel for item_component in report_item.item_components: component_dict = { "component_title": "", "component_authors": "", "component_publication_date": "", "component_data_type": "", "component_doi": "", "component_proprietary_id": "", "component_isbn": "", "component_print_issn": "", "component_online_issn": "", "component_uri": "" } if item_component.item_name: component_dict["component_title"] = item_component.item_name if item_component.data_type: component_dict["component_data_type"] = item_component.data_type # Authors authors_str = "" item_contributor: ItemContributorModel for item_contributor in item_component.item_contributors: if item_contributor.item_type == "Author": authors_str += f"{item_contributor.name}" if item_contributor.identifier: authors_str += f" ({item_contributor.identifier})" authors_str += "; " authors_str.rstrip("; ") if authors_str: component_dict["component_authors"] = authors_str # Publication date item_date: TypeValueModel for item_date in item_component.item_dates: if item_date.item_type == "Publication_Date" or item_date.item_type == "Pub_Date": component_dict["component_publication_date"] = item_date.value # Component IDs item_id: TypeValueModel for item_id in item_component.item_ids: item_type = item_id.item_type if item_type == "DOI": component_dict["component_doi"] = item_id.value elif item_type == "Proprietary" or item_type == "Proprietary_ID": component_dict["component_proprietary_id"] = item_id.value elif item_type == "ISBN": component_dict["component_isbn"] = item_id.value elif item_type == "Print_ISSN": component_dict["component_print_issn"] = item_id.value elif item_type == "Online_ISSN": component_dict["component_online_issn"] = item_id.value elif item_type == "URI": component_dict["component_uri"] = item_id.value components.append(component_dict) for metric_type in metric_row_dict: metric_row = metric_row_dict[metric_type] if major_report_type == MajorReportType.ITEM: if len(components) > 0 and \ (metric_type == "Total_Item_Investigations" or metric_type == "Total_Item_Requests"): for component in components: row = copy.copy(metric_row) row.component_title = component["component_title"] row.component_authors = component["component_authors"] row.component_publication_date = component["component_publication_date"] row.component_data_type = component["component_data_type"] row.component_doi = component["component_doi"] row.component_proprietary_id = component["component_proprietary_id"] row.component_isbn = component["component_isbn"] row.component_print_issn = component["component_print_issn"] row.component_online_issn = component["component_online_issn"] row.component_uri = component["component_uri"] report_rows.append(row) else: report_rows.append(metric_row) else: report_rows.append(metric_row) report_rows = self.sort_rows(report_rows, major_report_type) self.save_tsv_files(report_model.report_header, report_rows) @staticmethod def sort_rows(report_rows: list, major_report_type: MajorReportType) -> list: """Sorts the rows of the report :param report_rows: The report's rows :param major_report_type: The major report type of this report type """ if major_report_type == MajorReportType.PLATFORM: return sorted(report_rows, key=lambda row: row.platform.lower()) elif major_report_type == MajorReportType.DATABASE: return sorted(report_rows, key=lambda row: row.database.lower()) elif major_report_type == MajorReportType.TITLE: return sorted(report_rows, key=lambda row: (row.title.lower(), row.yop)) elif major_report_type == MajorReportType.ITEM: return sorted(report_rows, key=lambda row: row.item.lower()) def save_tsv_files(self, report_header, report_rows: list): """Saves the TSV file in the target directories :param report_header: The SUSHI report header model :param report_rows: The report's rows """ report_type = report_header.report_id major_report_type = report_header.major_report_type if self.is_yearly: file_dir = GeneralUtils.get_yearly_file_dir(self.save_dir, self.vendor.name, self.begin_date) file_name = GeneralUtils.get_yearly_file_name(self.vendor.name, self.report_type, self.begin_date) elif self.is_special: file_dir = GeneralUtils.get_special_file_dir(self.save_dir, self.vendor.name) file_name = GeneralUtils.get_special_file_name(self.vendor.name, self.report_type, self.begin_date, self.end_date) else: file_dir = GeneralUtils.get_other_file_dir(self.save_dir, self.vendor.name) file_name = GeneralUtils.get_other_file_name(self.vendor.name, self.report_type, self.begin_date, self.end_date) # Save user tsv file if not path.isdir(file_dir): makedirs(file_dir) file_path = f"{file_dir}{file_name}" file = open(file_path, 'w', encoding="utf-8", newline='') if self.is_yearly and self.is_master: self.add_report_header_to_file(report_header, file, False) else: self.add_report_header_to_file(report_header, file, True) if not self.add_report_rows_to_file(report_type, report_rows, self.begin_date, self.end_date, file, False, self.is_special, self.special_options): self.process_result.completion_status = CompletionStatus.WARNING file.close() self.process_result.file_name = file_name self.process_result.file_dir = file_dir self.process_result.file_path = file_path self.process_result.year = self.begin_date.toString('yyyy') # Save protected tsv file if self.is_yearly: protected_file_dir = GeneralUtils.get_yearly_file_dir(PROTECTED_DATABASE_FILE_DIR, self.vendor.name, self.begin_date) if not path.isdir(protected_file_dir): makedirs(protected_file_dir) if platform.system() == "Windows": ctypes.windll.kernel32.SetFileAttributesW(PROTECTED_DATABASE_FILE_DIR, 2) # Hide folder protected_file_path = f"{protected_file_dir}{file_name}" protected_file = open(protected_file_path, 'w', encoding="utf-8", newline='') self.add_report_header_to_file(report_header, protected_file, True) self.add_report_rows_to_file(report_type, report_rows, self.begin_date, self.end_date, protected_file, True) protected_file.close() self.process_result.protected_file_path = protected_file_path @staticmethod def add_report_header_to_file(report_header: ReportHeaderModel, file, include_attributes: bool): """Adds the report header to a TSV file :param report_header: The report header model :param file: The TSV file to write to :param include_attributes: Include the Report_Attributes value """ tsv_writer = csv.writer(file, delimiter='\t') tsv_writer.writerow(["Report_Name", report_header.report_name]) tsv_writer.writerow(["Report_ID", report_header.report_id]) tsv_writer.writerow(["Release", report_header.release]) tsv_writer.writerow(["Institution_Name", report_header.institution_name]) institution_ids_str = "" for institution_id in report_header.institution_ids: institution_ids_str += f"{institution_id.value}; " tsv_writer.writerow(["Institution_ID", institution_ids_str.rstrip("; ")]) metric_types_str = "" reporting_period_str = "" report_filters_str = "" for report_filter in report_header.report_filters: if report_filter.name == "Metric_Type": metric_types_str += f"{report_filter.value}; " elif report_filter.name == "Begin_Date" or report_filter.name == "End_Date": reporting_period_str += f"{report_filter.name}={report_filter.value}; " else: report_filters_str += f"{report_filter.name}={report_filter.value}; " tsv_writer.writerow(["Metric_Types", metric_types_str.replace("|", "; ").rstrip("; ")]) tsv_writer.writerow(["Report_Filters", report_filters_str.rstrip("; ")]) report_attributes_str = "" if include_attributes: for report_attribute in report_header.report_attributes: report_attributes_str += f"{report_attribute.name}={report_attribute.value}; " tsv_writer.writerow(["Report_Attributes", report_attributes_str.rstrip("; ")]) exceptions_str = "" for exception in report_header.exceptions: exceptions_str += f"{exception.code}: {exception.message} ({exception.data}); " tsv_writer.writerow(["Exceptions", exceptions_str.rstrip("; ")]) tsv_writer.writerow(["Reporting_Period", reporting_period_str.rstrip("; ")]) tsv_writer.writerow(["Created", report_header.created]) tsv_writer.writerow(["Created_By", report_header.created_by]) tsv_writer.writerow([]) @staticmethod def add_report_rows_to_file(report_type: str, report_rows: list, begin_date: QDate, end_date: QDate, file, include_all_attributes: bool, is_special: bool = False, special_options: SpecialReportOptions = None) -> bool: """Adds the report's rows to a TSV file :param report_type: The report type :param report_rows: The report's rows :param begin_date: The first date in the report :param end_date: The last date in the report :param file: The TSV file to write to :param include_all_attributes: Option to include all possible attributes for this report type to the report :param is_special: If this is a special report :param special_options: The special options if this is a special report """ column_names = [] row_dicts = [] if report_type == "PR": column_names += ["Platform"] if is_special: special_options_dict = special_options.__dict__ if special_options_dict["data_type"][0]: column_names.append("Data_Type") if special_options_dict["access_method"][0]: column_names.append("Access_Method") elif include_all_attributes: column_names.append("Data_Type") column_names.append("Access_Method") row: ReportRow for row in report_rows: row_dict = {"Platform": row.platform} if is_special: special_options_dict = special_options.__dict__ if special_options_dict["data_type"][0]: row_dict["Data_Type"] = row.data_type if special_options_dict["access_method"][0]: row_dict["Access_Method"] = row.access_method if not special_options_dict["exclude_monthly_details"][0]: row_dict.update(row.month_counts) else: if include_all_attributes: row_dict["Data_Type"] = row.data_type row_dict["Access_Method"] = row.access_method row_dict.update(row.month_counts) row_dict.update({"Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count}) row_dicts.append(row_dict) elif report_type == "PR_P1": column_names += ["Platform"] row: ReportRow for row in report_rows: row_dict = {"Platform": row.platform, "Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count} row_dict.update(row.month_counts) row_dicts.append(row_dict) elif report_type == "DR": column_names += ["Database", "Publisher", "Publisher_ID", "Platform", "Proprietary_ID"] if is_special: special_options_dict = special_options.__dict__ if special_options_dict["data_type"][0]: column_names.append("Data_Type") if special_options_dict["access_method"][0]: column_names.append("Access_Method") elif include_all_attributes: column_names.append("Data_Type") column_names.append("Access_Method") row: ReportRow for row in report_rows: row_dict = {"Database": row.database, "Publisher": row.publisher, "Publisher_ID": row.publisher_id, "Platform": row.platform, "Proprietary_ID": row.proprietary_id} if is_special: special_options_dict = special_options.__dict__ if special_options_dict["data_type"][0]: row_dict["Data_Type"] = row.data_type if special_options_dict["access_method"][0]: row_dict["Access_Method"] = row.access_method if not special_options_dict["exclude_monthly_details"][0]: row_dict.update(row.month_counts) else: if include_all_attributes: row_dict["Data_Type"] = row.data_type row_dict["Access_Method"] = row.access_method row_dict.update(row.month_counts) row_dict.update({"Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count}) row_dicts.append(row_dict) elif report_type == "DR_D1" or report_type == "DR_D2": column_names += ["Database", "Publisher", "Publisher_ID", "Platform", "Proprietary_ID"] row: ReportRow for row in report_rows: row_dict = {"Database": row.database, "Publisher": row.publisher, "Publisher_ID": row.publisher_id, "Platform": row.platform, "Proprietary_ID": row.proprietary_id, "Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count} row_dict.update(row.month_counts) row_dicts.append(row_dict) elif report_type == "TR": column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "ISBN", "Print_ISSN", "Online_ISSN", "URI"] if is_special: special_options_dict = special_options.__dict__ if special_options_dict["data_type"][0]: column_names.append("Data_Type") if special_options_dict["section_type"][0]: column_names.append("Section_Type") if special_options_dict["yop"][0]: column_names.append("YOP") if special_options_dict["access_type"][0]: column_names.append("Access_Type") if special_options_dict["access_method"][0]: column_names.append("Access_Method") elif include_all_attributes: column_names.append("Data_Type") column_names.append("Section_Type") column_names.append("YOP") column_names.append("Access_Type") column_names.append("Access_Method") row: ReportRow for row in report_rows: row_dict = {"Title": row.title, "Publisher": row.publisher, "Publisher_ID": row.publisher_id, "Platform": row.platform, "DOI": row.doi, "Proprietary_ID": row.proprietary_id, "ISBN": row.isbn, "Print_ISSN": row.print_issn, "Online_ISSN": row.online_issn, "URI": row.uri} if is_special: special_options_dict = special_options.__dict__ if special_options_dict["data_type"][0]: row_dict["Data_Type"] = row.data_type if special_options_dict["section_type"][0]: row_dict["Section_Type"] = row.section_type if special_options_dict["yop"][0]: row_dict["YOP"] = row.yop if special_options_dict["access_type"][0]: row_dict["Access_Type"] = row.access_type if special_options_dict["access_method"][0]: row_dict["Access_Method"] = row.access_method if not special_options_dict["exclude_monthly_details"][0]: row_dict.update(row.month_counts) else: if include_all_attributes: row_dict["Data_Type"] = row.data_type row_dict["Section_Type"] = row.section_type row_dict["YOP"] = row.yop row_dict["Access_Type"] = row.access_type row_dict["Access_Method"] = row.access_method row_dict.update(row.month_counts) row_dict.update({"Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count}) row_dicts.append(row_dict) elif report_type == "TR_B1" or report_type == "TR_B2": column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "ISBN", "Print_ISSN", "Online_ISSN", "URI", "YOP"] row: ReportRow for row in report_rows: row_dict = {"Title": row.title, "Publisher": row.publisher, "Publisher_ID": row.publisher_id, "Platform": row.platform, "DOI": row.doi, "Proprietary_ID": row.proprietary_id, "ISBN": row.isbn, "Print_ISSN": row.print_issn, "Online_ISSN": row.online_issn, "URI": row.uri, "YOP": row.yop, "Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count} row_dict.update(row.month_counts) row_dicts.append(row_dict) elif report_type == "TR_B3": column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "ISBN", "Print_ISSN", "Online_ISSN", "URI", "YOP", "Access_Type"] row: ReportRow for row in report_rows: row_dict = {"Title": row.title, "Publisher": row.publisher, "Publisher_ID": row.publisher_id, "Platform": row.platform, "DOI": row.doi, "Proprietary_ID": row.proprietary_id, "ISBN": row.isbn, "Print_ISSN": row.print_issn, "Online_ISSN": row.online_issn, "URI": row.uri, "YOP": row.yop, "Access_Type": row.access_type, "Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count} row_dict.update(row.month_counts) row_dicts.append(row_dict) elif report_type == "TR_J1" or report_type == "TR_J2": column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "Print_ISSN", "Online_ISSN", "URI"] row: ReportRow for row in report_rows: row_dict = {"Title": row.title, "Publisher": row.publisher, "Publisher_ID": row.publisher_id, "Platform": row.platform, "DOI": row.doi, "Proprietary_ID": row.proprietary_id, "Print_ISSN": row.print_issn, "Online_ISSN": row.online_issn, "URI": row.uri, "Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count} row_dict.update(row.month_counts) row_dicts.append(row_dict) elif report_type == "TR_J3": column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "Print_ISSN", "Online_ISSN", "URI", "Access_Type"] row: ReportRow for row in report_rows: row_dict = {"Title": row.title, "Publisher": row.publisher, "Publisher_ID": row.publisher_id, "Platform": row.platform, "DOI": row.doi, "Proprietary_ID": row.proprietary_id, "Print_ISSN": row.print_issn, "Online_ISSN": row.online_issn, "URI": row.uri, "Access_Type": row.access_type, "Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count} row_dict.update(row.month_counts) row_dicts.append(row_dict) elif report_type == "TR_J4": column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "Print_ISSN", "Online_ISSN", "URI", "YOP"] row: ReportRow for row in report_rows: row_dict = {"Title": row.title, "Publisher": row.publisher, "Publisher_ID": row.publisher_id, "Platform": row.platform, "DOI": row.doi, "Proprietary_ID": row.proprietary_id, "Print_ISSN": row.print_issn, "Online_ISSN": row.online_issn, "URI": row.uri, "YOP": row.yop, "Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count} row_dict.update(row.month_counts) row_dicts.append(row_dict) elif report_type == "IR": column_names += ["Item", "Publisher", "Publisher_ID", "Platform"] if is_special: special_options_dict = special_options.__dict__ if special_options_dict["authors"][0]: column_names.append("Authors") if special_options_dict["publication_date"][0]: column_names.append("Publication_Date") if special_options_dict["article_version"][0]: column_names.append("Article_version") elif include_all_attributes: column_names.append("Authors") column_names.append("Publication_Date") column_names.append("Article_version") column_names += ["DOI", "Proprietary_ID", "ISBN", "Print_ISSN", "Online_ISSN", "URI"] if is_special: special_options_dict = special_options.__dict__ if special_options_dict["include_parent_details"][0]: column_names += ["Parent_Title", "Parent_Authors", "Parent_Publication_Date", "Parent_Article_Version", "Parent_Data_Type", "Parent_DOI", "Parent_Proprietary_ID", "Parent_ISBN", "Parent_Print_ISSN", "Parent_Online_ISSN", "Parent_URI"] if special_options_dict["include_component_details"][0]: column_names += ["Component_Title", "Component_Authors", "Component_Publication_Date", "Component_Data_Type", "Component_DOI", "Component_Proprietary_ID", "Component_ISBN", "Component_Print_ISSN", "Component_Online_ISSN", "Component_URI"] if special_options_dict["data_type"][0]: column_names.append("Data_Type") if special_options_dict["yop"][0]: column_names.append("YOP") if special_options_dict["access_type"][0]: column_names.append("Access_Type") if special_options_dict["access_method"][0]: column_names.append("Access_Method") elif include_all_attributes: column_names += ["Parent_Title", "Parent_Authors", "Parent_Publication_Date", "Parent_Article_Version", "Parent_Data_Type", "Parent_DOI", "Parent_Proprietary_ID", "Parent_ISBN", "Parent_Print_ISSN", "Parent_Online_ISSN", "Parent_URI"] column_names += ["Component_Title", "Component_Authors", "Component_Publication_Date", "Component_Data_Type", "Component_DOI", "Component_Proprietary_ID", "Component_ISBN", "Component_Print_ISSN", "Component_Online_ISSN", "Component_URI"] column_names.append("Data_Type") column_names.append("YOP") column_names.append("Access_Type") column_names.append("Access_Method") row: ReportRow for row in report_rows: row_dict = {"Item": row.item, "Publisher": row.publisher, "Publisher_ID": row.publisher_id, "Platform": row.platform, "DOI": row.doi, "Proprietary_ID": row.proprietary_id, "ISBN": row.isbn, "Print_ISSN": row.print_issn, "Online_ISSN": row.online_issn, "URI": row.uri} if is_special: special_options_dict = special_options.__dict__ if special_options_dict["authors"][0]: row_dict["Authors"] = row.authors if special_options_dict["publication_date"][0]: row_dict["Publication_Date"] = row.publication_date if special_options_dict["article_version"][0]: row_dict["Article_version"] = row.article_version if special_options_dict["include_parent_details"][0]: row_dict.update({"Parent_Title": row.parent_title, "Parent_Authors": row.parent_authors, "Parent_Publication_Date": row.parent_publication_date, "Parent_Article_Version": row.parent_article_version, "Parent_Data_Type": row.parent_data_type, "Parent_DOI": row.parent_doi, "Parent_Proprietary_ID": row.parent_proprietary_id, "Parent_ISBN": row.parent_isbn, "Parent_Print_ISSN": row.parent_print_issn, "Parent_Online_ISSN": row.parent_online_issn, "Parent_URI": row.parent_uri}) if special_options_dict["include_component_details"][0]: row_dict.update({"Component_Title": row.component_title, "Component_Authors": row.component_authors, "Component_Publication_Date": row.component_publication_date, "Component_Data_Type": row.component_data_type, "Component_DOI": row.component_doi, "Component_Proprietary_ID": row.component_proprietary_id, "Component_ISBN": row.component_isbn, "Component_Print_ISSN": row.component_print_issn, "Component_Online_ISSN": row.component_online_issn, "Component_URI": row.component_uri}) if special_options_dict["data_type"][0]: row_dict["Data_Type"] = row.data_type if special_options_dict["yop"][0]: row_dict["YOP"] = row.yop if special_options_dict["access_type"][0]: row_dict["Access_Type"] = row.access_type if special_options_dict["access_method"][0]: row_dict["Access_Method"] = row.access_method if not special_options_dict["exclude_monthly_details"][0]: row_dict.update(row.month_counts) else: if include_all_attributes: row_dict["Authors"] = row.authors row_dict["Publication_Date"] = row.publication_date row_dict["Article_version"] = row.article_version row_dict.update({"Parent_Title": row.parent_title, "Parent_Authors": row.parent_authors, "Parent_Publication_Date": row.parent_publication_date, "Parent_Article_Version": row.parent_article_version, "Parent_Data_Type": row.parent_data_type, "Parent_DOI": row.parent_doi, "Parent_Proprietary_ID": row.parent_proprietary_id, "Parent_ISBN": row.parent_isbn, "Parent_Print_ISSN": row.parent_print_issn, "Parent_Online_ISSN": row.parent_online_issn, "Parent_URI": row.parent_uri}) row_dict.update({"Component_Title": row.component_title, "Component_Authors": row.component_authors, "Component_Publication_Date": row.component_publication_date, "Component_Data_Type": row.component_data_type, "Component_DOI": row.component_doi, "Component_Proprietary_ID": row.component_proprietary_id, "Component_ISBN": row.component_isbn, "Component_Print_ISSN": row.component_print_issn, "Component_Online_ISSN": row.component_online_issn, "Component_URI": row.component_uri}) row_dict["Data_Type"] = row.data_type row_dict["YOP"] = row.yop row_dict["Access_Type"] = row.access_type row_dict["Access_Method"] = row.access_method row_dict.update(row.month_counts) row_dict.update({"Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count}) row_dicts.append(row_dict) elif report_type == "IR_A1": column_names += ["Item", "Publisher", "Publisher_ID", "Platform", "Authors", "Publication_Date", "Article_version", "DOI", "Proprietary_ID", "Print_ISSN", "Online_ISSN", "URI", "Parent_Title", "Parent_Authors", "Parent_Article_Version", "Parent_DOI", "Parent_Proprietary_ID", "Parent_Print_ISSN", "Parent_Online_ISSN", "Parent_URI", "Access_Type"] row: ReportRow for row in report_rows: row_dict = {"Item": row.item, "Publisher": row.publisher, "Publisher_ID": row.publisher_id, "Platform": row.platform, "Authors": row.authors, "Publication_Date": row.publication_date, "Article_version": row.article_version, "DOI": row.doi, "Proprietary_ID": row.proprietary_id, "Print_ISSN": row.print_issn, "Online_ISSN": row.online_issn, "URI": row.uri, "Parent_Title": row.parent_title, "Parent_Authors": row.parent_authors, "Parent_Article_Version": row.parent_article_version, "Parent_DOI": row.parent_doi, "Parent_Proprietary_ID": row.parent_proprietary_id, "Parent_Print_ISSN": row.parent_print_issn, "Parent_Online_ISSN": row.parent_online_issn, "Parent_URI": row.parent_uri, "Access_Type": row.access_type, "Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count} row_dict.update(row.month_counts) row_dicts.append(row_dict) elif report_type == "IR_M1": column_names += ["Item", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "URI"] row: ReportRow for row in report_rows: row_dict = {"Item": row.item, "Publisher": row.publisher, "Publisher_ID": row.publisher_id, "Platform": row.platform, "DOI": row.doi, "Proprietary_ID": row.proprietary_id, "URI": row.uri, "Metric_Type": row.metric_type, "Reporting_Period_Total": row.total_count} row_dict.update(row.month_counts) row_dicts.append(row_dict) column_names += ["Metric_Type", "Reporting_Period_Total"] if is_special: special_options_dict = special_options.__dict__ if not special_options_dict["exclude_monthly_details"][0]: column_names += get_month_years(begin_date, end_date) else: column_names += get_month_years(begin_date, end_date) tsv_dict_writer = csv.DictWriter(file, column_names, delimiter='\t') tsv_dict_writer.writeheader() if len(row_dicts) == 0: return False tsv_dict_writer.writerows(row_dicts) return True def save_json_file(self, json_string: str): """Saves a raw JSON file of the report :param json_string: The JSON string """ file_dir = f"{PROTECTED_DATABASE_FILE_DIR}_json/{self.begin_date.toString('yyyy')}/{self.vendor.name}/" file_name = f"{self.begin_date.toString('yyyy')}_{self.vendor.name}_{self.report_type}.json" file_path = f"{file_dir}{file_name}" if not path.isdir(file_dir): makedirs(file_dir) json_file = open(file_path, 'w', encoding="utf-8") json_file.write(json_string) json_file.close() def notify_worker_finished(self): """Notifies any listeners that this worker has finished""" self.worker_finished_signal.emit(self.worker_id) # El Psy Kongroo