| #!/usr/bin/env python3 |
| # Copyright 2024 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """A statistical analysis tool for fingerprint evaluation tool results. |
| """ |
| |
| from __future__ import annotations |
| |
| import argparse |
| import pathlib |
| import sys |
| from typing import Optional |
| |
| import bootstrap |
| from experiment import Experiment |
| from fpc_bet_results import FPCBETResults |
| import fpsutils |
| import numpy as np |
| import pandas as pd |
| import plotly.express as px |
| import plotly.graph_objs as go |
| from report_pandoc import Report2 |
| import scipy.stats as st |
| from test_case import TestCase |
| from tqdm.autonotebook import tqdm # Auto detect notebook or console. |
| |
| |
| def print_far_value( |
| val: float, |
| k_comparisons: list[int] = [20, 50, 100, 500], |
| ) -> str: |
| """Print the science notation""" |
| str = f"{val:.4e}" |
| for c in k_comparisons: |
| str += f" = {val * 100 * c*1000:.3f}% of 1/{c}k" |
| # TODO: Make nice output like "1 in 234k". |
| return str |
| |
| |
| # Ultimately, we want to do a histogram over the entire FAR/FRR Decision |
| # dataset, but doing so directly with the large DataFrame is much too slow |
| # and will actually hang plotly. We are mimicking the following histogram |
| # operation: |
| # go.Histogram(histfunc="sum", x=far['EnrollUser'], y=far['Decision']) |
| # |
| # A similar method might be using the following, which runs in about 300ms: |
| # far[['EnrollUser', 'Decision']].groupby(['EnrollUser']).sum() |
| # |
| # The fastest method is by reverse constructing the complete counts table |
| # by using the pre-aggregated fa_table. This runs in about 66ms, which is |
| # primarily the time to run exp.user_list(). |
| |
| |
| def fa_count_figure( |
| exp: Experiment, |
| cols: list[Experiment.TableCol], |
| title: str, |
| xaxis_title: str, |
| ) -> go.Figure: |
| fa_counts = pd.DataFrame({c.value: exp.fa_counts_by(c) for c in cols}) |
| fa_counts.rename( |
| columns={c.value: f"FA Counts by {c.value}" for c in cols}, inplace=True |
| ) |
| non_zero_labels = fa_counts.loc[(fa_counts > 0).any(axis=1)].index |
| # non_zero_labels = list(non_zero_counts_enroll.index) + \ |
| # list(non_zero_counts_verify.index) |
| # non_zero_labels.sort() |
| |
| # It is nice to keep the blank space between non-zero bars to be able to |
| # identify possible abnormal clusters of high false acceptance users. |
| fig = px.bar( |
| fa_counts, |
| # pattern_shape='variable', pattern_shape_sequence=['/', '\\'], |
| # text_auto=True, |
| # labels={'EnrollUser': 'FA Counts by EnrollUser', |
| # 'VerifyUser': 'FA Counts by VerifyUser'} |
| # orientation='h', |
| ) |
| # fig.update_xaxes(type='category') |
| # fig.update_layout(barmode='overlay') |
| # fig.update_layout(barmode='group') |
| # Reduce opacity to see both histograms |
| # fig.update_traces(opacity=0.75) |
| # fig.update_traces(opacity=0.50) |
| # fig.update_traces(marker_size=10) |
| # fig.update_traces(marker_line_color = 'blue', marker_line_width = 0.25) |
| fig.update_layout( |
| # title_text='False Accepts by User ID of Enroll and Verify', |
| title=title, |
| # xaxis_title_text='User ID', |
| xaxis_title=xaxis_title, |
| yaxis_title_text="False Accept Count", |
| legend_title="", |
| barmode="group", |
| # barmode='overlay', |
| # bargap=0.2, # gap between bars of adjacent location coordinates |
| # bargroupgap=0.1 # gap between bars of the same location coordinates |
| bargap=0.0, |
| bargroupgap=0.0, |
| xaxis=dict(type="category", tickmode="array", tickvals=non_zero_labels), |
| ) |
| fig.update_layout( |
| legend=dict( |
| # orientation='h', |
| yanchor="top", |
| y=0.99, |
| xanchor="left", |
| x=0.01, |
| ) |
| ) |
| return fig |
| |
| |
| def fr_count_figure( |
| exp: Experiment, |
| cols: list[Experiment.TableCol], |
| title: str, |
| xaxis_title: str, |
| ) -> go.Figure: |
| fr_counts = pd.DataFrame({c.value: exp.fr_counts_by(c) for c in cols}) |
| fr_counts.rename( |
| columns={c.value: f"FR Counts by {c.value}" for c in cols}, inplace=True |
| ) |
| non_zero_labels = fr_counts.loc[(fr_counts > 0).any(axis=1)].index |
| # non_zero_labels = list(non_zero_counts_enroll.index) + \ |
| # list(non_zero_counts_verify.index) |
| # non_zero_labels.sort() |
| |
| fr_tests_total = exp.fr_trials_count() |
| fr_category_size = len(fr_counts.index) |
| fr_per_category = float(fr_tests_total) / float(fr_category_size) |
| percents = ( |
| np.array(fr_counts.values, dtype=float) / fr_per_category |
| ) * 100.0 |
| |
| # It is nice to keep the blank space between non-zero bars to be able to |
| # identify possible abnormal clusters of high false acceptance users. |
| # fig = go.Figure() |
| # fig.add_trace(go.Bar(x=fr_counts.index, y=fr_counts.values[:, 0])) |
| fig: go.Figure = px.bar( |
| fr_counts, |
| # pattern_shape='variable', pattern_shape_sequence=['/', '\\'], |
| # text_auto=True, |
| # labels={'EnrollUser': 'FA Counts by EnrollUser', |
| # 'VerifyUser': 'FA Counts by VerifyUser'} |
| # orientation='h', |
| hover_data={ |
| "percent": (":.3f", percents[:, 0]), |
| }, |
| ) |
| # fr_tests_total = exp.fr_trials_count() |
| # fr_category_size = len(fr_counts.index) |
| # fr_per_category = float(fr_tests_total) / float(fr_category_size) |
| # percents = (np.array(fr_counts.values, dtype=float) / fr_per_category) * 100.0 |
| # fr_percents = pd.Series(percents, index=fr_counts.index, name='Percents') |
| # fig.add_trace(go.Bar(x=fr_counts.index, y=percents[:, 0], yaxis='y2')) |
| |
| # fig.update_xaxes(type='category') |
| # fig.update_layout(barmode='overlay') |
| # fig.update_layout(barmode='group') |
| # Reduce opacity to see both histograms |
| # fig.update_traces(opacity=0.75) |
| # fig.update_traces(opacity=0.50) |
| # fig.update_traces(marker_size=10) |
| # fig.update_traces(marker_line_color = 'blue', marker_line_width = 0.25) |
| fig.update_layout( |
| # title_text='False Rejects by User ID of Enroll and Verify', |
| title=title, |
| # xaxis_title_text='User ID', |
| xaxis_title=xaxis_title, |
| # yaxis_title_text='False Rejects Count', |
| legend_title="", |
| barmode="group", |
| # barmode='overlay', |
| # bargap=0.2, # gap between bars of adjacent location coordinates |
| # bargroupgap=0.1 # gap between bars of the same location coordinates |
| bargap=0.0, |
| bargroupgap=0.0, |
| xaxis=dict(type="category", tickmode="array", tickvals=non_zero_labels), |
| yaxis=dict( |
| title="False Rejects Count", |
| showline=True, |
| ), |
| yaxis2=dict( |
| title="False Reject Percent", |
| side="right", |
| showline=True, |
| ), |
| ) |
| fig.update_layout( |
| legend=dict( |
| # orientation='h', |
| yanchor="top", |
| y=0.99, |
| xanchor="left", |
| x=0.01, |
| ) |
| ) |
| return fig |
| |
| |
| def cmd_report(opts: argparse.Namespace) -> int: |
| """Conduct a full analysis of all test cases and generate a final report.""" |
| user_groups_csv: Optional[pathlib.Path] = opts.user_groups_csv |
| testcases_decisions_dir: pathlib.Path = opts.testcases_decisions_dir |
| analysis_dir: pathlib.Path = opts.analysis_dir |
| |
| if not user_groups_csv: |
| user_groups_csv = testcases_decisions_dir / "User_groups.csv" |
| |
| analysis_dir.mkdir(exist_ok=True) |
| source_dir = pathlib.Path(__file__).parent |
| rpt = Report2(analysis_dir, source_dir / "templates") |
| |
| ################# Import Data From BET Results ################# |
| |
| print("# Read in data") |
| |
| bet = FPCBETResults(testcases_decisions_dir) |
| |
| # FIXME: Only enable one test case for speed of testing. |
| # test_cases = [FPCBETResults.TestCase.TUDisabled] |
| test_cases = FPCBETResults.TestCase.all() |
| |
| far_decisions = bet.read_files( |
| list( |
| zip( |
| test_cases, |
| [FPCBETResults.TableType.FAR_Decision] * len(test_cases), |
| ) |
| ) |
| ) |
| |
| frr_decisions = bet.read_files( |
| list( |
| zip( |
| test_cases, |
| [FPCBETResults.TableType.FRR_Decision] * len(test_cases), |
| ) |
| ) |
| ) |
| |
| exps = { |
| test_cases[i]: Experiment( |
| num_verification=80, |
| num_fingers=6, |
| num_users=72, |
| far_decisions=far, |
| frr_decisions=frr, |
| ) |
| for i, (far, frr) in enumerate(zip(far_decisions, frr_decisions)) |
| } |
| |
| for tc in exps: |
| exps[tc].add_groups_from_csv(user_groups_csv) |
| |
| ################# Generate Report Test cases ################# |
| |
| print("# Setup report test cases") |
| |
| rpt_tc = { |
| tc: rpt.test_case_add(str(tc), tc.description()) for tc in test_cases |
| } |
| |
| # Ultimately, we want to do a histogram over the entire FAR/FRR Decision |
| # dataset, but doing so directly with the large DataFrame is much too slow |
| # and will actually hang plotly. We are mimicking the following histogram |
| # operation: |
| # go.Histogram(histfunc="sum", x=far['EnrollUser'], y=far['Decision']) |
| # |
| # A similar method might be using the following, which runs in about 300ms: |
| # far[['EnrollUser', 'Decision']].groupby(['EnrollUser']).sum() |
| # |
| # The fastest method is by reverse constructing the complete counts table |
| # by using the pre-aggregated fa_table. This runs in about 66ms, which is |
| # primarily the time to run exp.user_list(). |
| |
| ################# Histograms ################# |
| |
| print("# Add main histograms to report") |
| |
| for tc in test_cases: |
| exp = exps[tc] |
| section = rpt_tc[tc].add_subsection("hist") |
| |
| # A high FA count for an EnrollUser would indicate some template(s) for a given |
| # user allows more false accepts from other users. |
| # A high FA count for a VerifyUser would indicate that some match attempts |
| # with this user's fingers yields more false accepts. |
| |
| # User |
| fig = fa_count_figure( |
| exp, |
| [Experiment.TableCol.Enroll_User, Experiment.TableCol.Verify_User], |
| "False Accepts by User ID of Enroll and Verify", |
| "User ID", |
| ) |
| section.add_figure( |
| "FA_by_User", "False Accepts by User ID of Enroll and Verify.", fig |
| ) |
| fig = fr_count_figure( |
| exp, |
| [Experiment.TableCol.Verify_User], |
| "False Rejects by User ID", |
| "User ID", |
| ) |
| section.add_figure("FR_by_User", "False Rejects by User ID.", fig) |
| |
| # Finger |
| fig = fa_count_figure( |
| exp, |
| [ |
| Experiment.TableCol.Enroll_Finger, |
| Experiment.TableCol.Verify_Finger, |
| ], |
| "False Accepts by Finger ID of Enroll and Verify", |
| "Finger ID", |
| ) |
| section.add_figure( |
| "FA_by_Finger", |
| "False Accepts by Finger ID of Enroll and Verify.", |
| fig, |
| ) |
| fig = fr_count_figure( |
| exp, |
| [Experiment.TableCol.Verify_Finger], |
| "False Rejects by Finger ID", |
| "Finger ID", |
| ) |
| section.add_figure("FR_by_Finger", "False Rejects by Finger ID.", fig) |
| |
| # Sample |
| # Keep in mind that different test cases may select different samples |
| # for verification. |
| fig = fa_count_figure( |
| exp, |
| [Experiment.TableCol.Verify_Sample], |
| "False Accepts by Verify Sample ID", |
| "Sample ID", |
| ) |
| section.add_figure( |
| "FA_by_Sample", "False Accepts by Verify Sample ID.", fig |
| ) |
| fig = fr_count_figure( |
| exp, |
| [Experiment.TableCol.Verify_Sample], |
| "False Rejects by Sample ID. " |
| "Keep in mind that different test cases may use " |
| "different samples for verification.", |
| "Sample ID", |
| ) |
| fr_counts = exp.fr_counts_by(Experiment.TableCol.Verify_Sample) |
| line = st.linregress(fr_counts.index, fr_counts.values) |
| line_x = np.array(fr_counts.index) |
| line = st.linregress(line_x, np.array(fr_counts.values)) |
| line_y = line.slope * line_x + line.intercept |
| fig.add_trace(go.Line(x=line_x, y=line_y, name="Linear Regression")) |
| section.add_figure("FR_by_Sample", "False Rejects by Sample ID.", fig) |
| |
| # Group |
| fig = fa_count_figure( |
| exp, |
| [ |
| Experiment.TableCol.Enroll_Group, |
| Experiment.TableCol.Verify_Group, |
| ], |
| "False Accepts by Group of Enroll and Verify", |
| "Group", |
| ) |
| section.add_figure( |
| "FA_by_Group", "False Accepts by Group of Enroll and Verify.", fig |
| ) |
| fig = fr_count_figure( |
| exp, |
| [Experiment.TableCol.Verify_Group], |
| "False Rejects by Group", |
| "Group", |
| ) |
| section.add_figure("FR_by_Group", "False Rejects by Group.", fig) |
| |
| # rpt_tc[DISPLAY_TC].display(display) |
| |
| print("# Add remaining histograms to report") |
| |
| ### FA_by_User |
| |
| s1 = { |
| "EnrollUser_" |
| + tc.name: exps[tc].fa_counts_by(Experiment.TableCol.Enroll_User) |
| for tc in test_cases |
| } |
| s2 = { |
| "VerifyUser_" |
| + tc.name: exps[tc].fa_counts_by(Experiment.TableCol.Verify_User) |
| for tc in test_cases |
| } |
| df = pd.DataFrame(s1 | s2) |
| # [tc.name for tc in test_cases] |
| fig = px.bar(df) |
| fig.update_layout( |
| title="False Accepts by User", |
| xaxis_title="User", |
| yaxis_title_text="Count", |
| legend_title="Category + Test Case", |
| barmode="group", |
| # height=2049, |
| # barmode='overlay', |
| # bargap=0.2, # gap between bars of adjacent location coordinates |
| # bargroupgap=0.1 # gap between bars of the same location coordinates |
| bargap=0.0, |
| bargroupgap=0.0, |
| xaxis=dict( |
| type="category", |
| # tickmode='array', |
| # tickvals=non_zero_labels, |
| ), |
| ) |
| fig.update_layout( |
| legend=dict( |
| # orientation='h', |
| yanchor="top", |
| y=0.99, |
| xanchor="left", |
| x=0.01, |
| ) |
| ) |
| # fig.show() |
| rpt.overall_section().add_figure("FA_by_User", "False Accepts by User", fig) |
| |
| ### FR_by_User |
| |
| df = pd.DataFrame( |
| { |
| tc.name: exps[tc].fr_counts_by(Experiment.TableCol.Verify_User) |
| for tc in test_cases |
| } |
| ) |
| # [tc.name for tc in test_cases] |
| fig = px.bar( |
| df, |
| # orientation='h', |
| ) |
| fig.update_layout( |
| title="False Rejects by User", |
| xaxis_title="User", |
| yaxis_title_text="Count", |
| legend_title="Test Case", |
| barmode="group", |
| # height=2049, |
| # barmode='overlay', |
| # bargap=0.2, # gap between bars of adjacent location coordinates |
| # bargroupgap=0.1 # gap between bars of the same location coordinates |
| bargap=0.0, |
| bargroupgap=0.0, |
| xaxis=dict( |
| type="category", |
| # tickmode='array', |
| # tickvals=non_zero_labels, |
| ), |
| ) |
| fig.update_layout( |
| legend=dict( |
| # orientation='h', |
| yanchor="top", |
| y=0.99, |
| xanchor="left", |
| x=0.01, |
| ) |
| ) |
| # fig.show() |
| rpt.overall_section().add_figure("FR_by_User", "False Rejects by User", fig) |
| |
| ### FA_by_Sample |
| |
| df = pd.DataFrame( |
| { |
| tc.name: exps[tc].fa_counts_by(Experiment.TableCol.Verify_Sample) |
| for tc in test_cases |
| } |
| ) |
| # [tc.name for tc in test_cases] |
| fig = px.bar( |
| df, |
| # orientation='h', |
| ) |
| fig.update_layout( |
| # title_text='False Rejects by User ID of Enroll and Verify', |
| title="False Accepts by Sample", |
| xaxis_title="Group", |
| yaxis_title_text="Count", |
| legend_title="Test Case", |
| barmode="group", |
| # height=2049, |
| # barmode='overlay', |
| # bargap=0.2, # gap between bars of adjacent location coordinates |
| # bargroupgap=0.1 # gap between bars of the same location coordinates |
| bargap=0.0, |
| bargroupgap=0.0, |
| # xaxis=dict(type='category', |
| # tickmode='array', |
| # tickvals=non_zero_labels), |
| xaxis=dict( |
| type="category", |
| # tickmode='array', |
| # tickvals=non_zero_labels, |
| ), |
| ) |
| # fig.update_layout( |
| # # title_text='False Accepts by User ID of Enroll and Verify', |
| # title=title, |
| # # xaxis_title_text='User ID', |
| # xaxis_title=xaxis_title, |
| # yaxis_title_text='False Accept Count', |
| # legend_title='', |
| # barmode='group', |
| # # barmode='overlay', |
| # # bargap=0.2, # gap between bars of adjacent location coordinates |
| # # bargroupgap=0.1 # gap between bars of the same location coordinates |
| # bargap=0.0, |
| # bargroupgap=0.0, |
| # xaxis=dict(type='category', |
| # tickmode='array', |
| # tickvals=non_zero_labels), |
| # ) |
| fig.update_layout( |
| legend=dict( |
| # orientation='h', |
| yanchor="top", |
| y=0.99, |
| xanchor="left", |
| x=0.01, |
| ) |
| ) |
| # fig.show() |
| rpt.overall_section().add_figure( |
| "FA_by_Sample", "False Accepts by Sample", fig |
| ) |
| |
| ### FA_by_Finger |
| |
| s1 = { |
| "EnrollFinger_" |
| + tc.name: exps[tc].fa_counts_by(Experiment.TableCol.Enroll_Finger) |
| for tc in test_cases |
| } |
| s2 = { |
| "VerifyFinger_" |
| + tc.name: exps[tc].fa_counts_by(Experiment.TableCol.Verify_Finger) |
| for tc in test_cases |
| } |
| df = pd.DataFrame(s1 | s2) |
| # [tc.name for tc in test_cases] |
| fig = px.bar(df) |
| fig.update_layout( |
| title="False Accepts by Finger", |
| xaxis_title="Finger", |
| yaxis_title_text="Count", |
| legend_title="Category + Test Case", |
| barmode="group", |
| ) |
| # fig.show() |
| rpt.overall_section().add_figure( |
| "FA_by_Finger", "False Accepts by Finger", fig |
| ) |
| |
| ### FR_by_Finger |
| |
| df = pd.DataFrame( |
| { |
| tc.name: exps[tc].fr_counts_by(Experiment.TableCol.Verify_Finger) |
| for tc in test_cases |
| } |
| ) |
| # [tc.name for tc in test_cases] |
| fig = px.bar(df) |
| fig.update_layout( |
| title="False Rejects by Finger", |
| xaxis_title="Finger", |
| yaxis_title_text="Count", |
| legend_title="Test Case", |
| barmode="group", |
| ) |
| # fig.show() |
| rpt.overall_section().add_figure( |
| "FR_by_Finger", "False Rejects by Finger", fig |
| ) |
| |
| ### FA_by_Group |
| |
| s1 = { |
| "EnrollGroup_" |
| + tc.name: exps[tc].fa_counts_by(Experiment.TableCol.Enroll_Group) |
| for tc in test_cases |
| } |
| s2 = { |
| "VerifyGroup_" |
| + tc.name: exps[tc].fa_counts_by(Experiment.TableCol.Verify_Group) |
| for tc in test_cases |
| } |
| df = pd.DataFrame(s1 | s2) |
| # [tc.name for tc in test_cases] |
| fig = px.bar(df) |
| fig.update_layout( |
| title="False Accepts by Group", |
| xaxis_title="Group", |
| yaxis_title_text="Count", |
| legend_title="Category + Test Case", |
| barmode="group", |
| ) |
| # fig.show() |
| rpt.overall_section().add_figure( |
| "FA_by_Group", "False Accepts by Group", fig |
| ) |
| |
| ### FR_by_Group |
| |
| df = pd.DataFrame( |
| { |
| tc.name: exps[tc].fr_counts_by(Experiment.TableCol.Verify_Group) |
| for tc in test_cases |
| } |
| ) |
| # [tc.name for tc in test_cases] |
| fig = px.bar(df) |
| fig.update_layout( |
| # title_text='False Rejects by User ID of Enroll and Verify', |
| title="False Rejects by Group", |
| xaxis_title="Group", |
| yaxis_title_text="Count", |
| legend_title="Test Case", |
| barmode="group", |
| ) |
| # fig.show() |
| rpt.overall_section().add_figure( |
| "FR_by_Group", "False Rejects by Group", fig |
| ) |
| |
| ################# Bootstrap Sampling ################# |
| |
| print("# Run bootstrap samples") |
| |
| # 1000 samples is 95% |
| # 5000 samples is 99% |
| # BOOTSTRAP_SAMPLES = 1000 |
| # BOOTSTRAP_SAMPLES = 5000 # 5000 samples in 2 seconds (128 cores) |
| BOOTSTRAP_SAMPLES = 100000 # 100000 samples in 16 seconds (128 cores) |
| CONFIDENCE_PERCENT = 95 |
| FAR_THRESHOLD = 1 / 100000.0 |
| FRR_THRESHOLD = 10 / 100.0 |
| |
| far_boot_results: dict[TestCase, bootstrap.BootstrapResults] = dict() |
| frr_boot_results: dict[TestCase, bootstrap.BootstrapResults] = dict() |
| far_figures: dict[TestCase, go.Figure] = dict() |
| frr_figures: dict[TestCase, go.Figure] = dict() |
| for tc in test_cases: |
| print(f"Running Test Case {tc}.") |
| exp = exps[tc] |
| section = rpt_tc[tc].add_subsection("score") |
| info = section.add_data("Info") |
| |
| #### FAR #### |
| |
| # Run FAR bootstrap |
| boot = bootstrap.BootstrapFullFARHierarchy(exp, verbose=True) |
| # boot = bootstrap.BootstrapFARFlat(exp, verbose=True) |
| boot_results = boot.run( |
| num_samples=BOOTSTRAP_SAMPLES, |
| num_proc=0, |
| progress=lambda it, total: tqdm(it, total=total), |
| ) |
| far_boot_results[tc] = boot_results |
| # Showing raw values works because we take som many bootstrap samples, |
| # which fills in a lot of gaps in a "unique" value (bin_size=1) histogram. |
| bins, counts = fpsutils.discrete_hist(boot_results.samples()) |
| df = pd.DataFrame( |
| { |
| # X-Axes |
| "False Accepts in Bootstrap Sample": bins, |
| "FAR": np.array(bins, dtype=float) / exp.fa_trials_count(), |
| # Y-Axes |
| "Number of Bootstrap Samples Observed": counts, |
| } |
| ) |
| fig = px.bar( |
| df, |
| x="FAR", |
| y="Number of Bootstrap Samples Observed", |
| hover_data=["False Accepts in Bootstrap Sample"], |
| title="Frequency of FAR in Bootstrap Samples", |
| ) |
| fig.update_layout(hovermode="x unified") |
| |
| ci_lower, ci_upper = boot_results.confidence_interval() |
| frr_ci_lower = ci_lower / exp.fa_trials_count() |
| frr_ci_upper = ci_upper / exp.fa_trials_count() |
| frr_mean = np.mean(boot_results.samples()) / exp.fa_trials_count() |
| frr_std = np.std(boot_results.samples()) / exp.fa_trials_count() |
| fig.add_vline( |
| x=frr_ci_lower, |
| annotation_text=f"lower {CONFIDENCE_PERCENT}%", |
| annotation_position="top right", |
| line_width=1, |
| line_dash="dash", |
| line_color="yellow", |
| ) |
| fig.add_vline( |
| x=frr_ci_upper, |
| annotation_text=f"upper {CONFIDENCE_PERCENT}%", |
| annotation_position="top left", |
| line_width=1, |
| line_dash="dash", |
| line_color="yellow", |
| ) |
| fig.add_vline( |
| x=frr_mean, |
| annotation_text="mean", |
| annotation_position="top left", |
| line_width=1, |
| line_dash="dash", |
| line_color="yellow", |
| ) |
| |
| # Enable this to compare against 1/50k. |
| # Enabling this has the undesirable side effect of shifting |
| # focus of the histogram plots to include 1/50k, which might |
| # be very far from the mean. |
| # fig.add_vline( |
| # x=1 / 50000.0, |
| # annotation_text="1/50", |
| # line_width=2, |
| # line_dash="dash", |
| # line_color="red", |
| # ) |
| fig.add_vline( |
| x=1 / 100000.0, |
| annotation_text="1/100k", |
| line_width=1, |
| line_dash="dash", |
| line_color="red", |
| ) |
| fig.add_vline( |
| x=1 / 200000.0, |
| annotation_text="1/200k", |
| line_width=1, |
| line_dash="dash", |
| line_color="red", |
| ) |
| far_figures[tc] = fig |
| |
| section.add_figure( |
| "FAR_Bootstrap", |
| "The hierarchical FAR bootstrap sampling histogram.", |
| fig, |
| ) |
| |
| info.set("FAR_Confidence", CONFIDENCE_PERCENT) |
| info.set("FAR_Trials", exp.fa_trials_count()) |
| info.set("FAR_False_Accepts", exp.fa_count()) |
| info.set("FAR_CI_Lower", frr_ci_lower) |
| info.set("FAR_CI_Upper", frr_ci_upper) |
| info.set("FAR_Mean", frr_mean) |
| info.set("FAR_Std", frr_std) |
| info.set("FAR_Threshold", f"1/{1 / (FAR_THRESHOLD*1000)}k") |
| info.set("FAR_Pass", frr_ci_upper < FAR_THRESHOLD) |
| |
| #### FRR #### |
| |
| # Run FRR bootstrap |
| boot = bootstrap.BootstrapFullFRRHierarchy(exp, verbose=True) |
| # boot = bootstrap.BootstrapFARFlat(exp, verbose=True) |
| boot_results = boot.run( |
| num_samples=BOOTSTRAP_SAMPLES, |
| num_proc=0, |
| progress=lambda it, total: tqdm(it, total=total), |
| ) |
| frr_boot_results[tc] = boot_results |
| |
| # Showing raw values works because we take som many bootstrap samples, |
| # which fills in a lot of gaps in a "unique" value (bin_size=1) histogram. |
| bins, counts = fpsutils.discrete_hist(boot_results.samples()) |
| df = pd.DataFrame( |
| { |
| # X-Axes |
| "False Accepts in Bootstrap Sample": bins, |
| "FRR": np.array(bins, dtype=float) / exp.fr_trials_count(), |
| # Y-Axes |
| "Number of Bootstrap Samples Observed": counts, |
| } |
| ) |
| fig = px.bar( |
| df, |
| x="FRR", |
| y="Number of Bootstrap Samples Observed", |
| hover_data=["False Accepts in Bootstrap Sample"], |
| title="Frequency of FRR in Bootstrap Samples", |
| ) |
| fig.update_layout( |
| hovermode="x unified", |
| xaxis=dict(tickformat="%"), |
| bargap=0.0, |
| bargroupgap=0.0, |
| ) |
| |
| ci_lower, ci_upper = boot_results.confidence_interval() |
| frr_ci_lower = ci_lower / exp.fr_trials_count() |
| frr_ci_upper = ci_upper / exp.fr_trials_count() |
| frr_mean = np.mean(boot_results.samples()) / exp.fr_trials_count() |
| frr_std = np.std(boot_results.samples()) / exp.fr_trials_count() |
| fig.add_vline( |
| x=frr_ci_lower, |
| annotation_text=f"lower {CONFIDENCE_PERCENT}%", |
| annotation_position="top right", |
| line_width=1, |
| line_dash="dash", |
| line_color="yellow", |
| ) |
| fig.add_vline( |
| x=frr_ci_upper, |
| annotation_text=f"upper {CONFIDENCE_PERCENT}%", |
| annotation_position="top left", |
| line_width=1, |
| line_dash="dash", |
| line_color="yellow", |
| ) |
| fig.add_vline( |
| x=frr_mean, |
| annotation_text="mean", |
| annotation_position="top left", |
| line_width=1, |
| line_dash="dash", |
| line_color="yellow", |
| ) |
| |
| fig.add_vline( |
| x=10 / 100, |
| annotation_text="10%", |
| line_width=2, |
| line_dash="dash", |
| line_color="red", |
| ) |
| |
| frr_figures[tc] = fig |
| |
| section.add_figure( |
| "FRR_Bootstrap", |
| "The hierarchical FRR bootstrap sampling histogram.", |
| fig, |
| ) |
| |
| info.set("FRR_Confidence", CONFIDENCE_PERCENT) |
| info.set("FRR_Trials", exp.fr_trials_count()) |
| info.set("FRR_False_Accepts", exp.fr_count()) |
| info.set("FRR_CI_Lower", frr_ci_lower) |
| info.set("FRR_CI_Upper", frr_ci_upper) |
| info.set("FRR_Mean", frr_mean) |
| info.set("FRR_Std", frr_std) |
| info.set("FRR_Threshold", f"{FRR_THRESHOLD * 100}%") |
| info.set("FRR_Pass", frr_ci_upper < FRR_THRESHOLD) |
| |
| #### Generate Report #### |
| |
| print("# Generate final report") |
| |
| rpt.generate( |
| { |
| # 'pdf', |
| # 'md', |
| "html", |
| } |
| ) |
| |
| print(f"View {(analysis_dir / 'index.html').absolute}") |
| |
| return 0 |
| |
| |
| def cmd_groups_discover(opts: argparse.Namespace) -> int: |
| """Discover the user-group mapping from a raw collection dir structure. |
| |
| Write this table out to a CSV file, which typically is called |
| User_groups.csv. |
| """ |
| src_collection_dir: pathlib.Path = opts.src_collection_dir |
| user_groups_csv: pathlib.Path = opts.user_groups_csv |
| |
| exp = Experiment(0, 0, 0) |
| exp.add_groups_from_collection_dir(src_collection_dir) |
| exp.user_groups_table_to_csv(user_groups_csv) |
| return 0 |
| |
| |
| def main(argv: list[str]) -> int: |
| parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) |
| subparsers = parser.add_subparsers( |
| dest="subcommand", required=True, title="subcommands" |
| ) |
| |
| # Parser for "report" subcommand. |
| parser_report = subparsers.add_parser("report", help=cmd_report.__doc__) |
| parser_report.set_defaults(func=cmd_report) |
| parser_report.add_argument( |
| "--user-groups-csv", |
| type=pathlib.Path, |
| help="Path to the user-group mapping CSV file. " |
| "(default: <testcases_decisions_dir>/User_groups.csv).", |
| ) |
| parser_report.add_argument( |
| "testcases_decisions_dir", |
| type=pathlib.Path, |
| help="Directory of directories that holds the matcher decisions for each test case", |
| ) |
| parser_report.add_argument( |
| "analysis_dir", |
| default="analysis", |
| type=pathlib.Path, |
| help="Directory to output the analysis report", |
| ) |
| |
| # Parser for "groups-discover" subcommand. |
| parser_groups_discover = subparsers.add_parser( |
| "groups-discover", help=cmd_groups_discover.__doc__ |
| ) |
| parser_groups_discover.set_defaults(func=cmd_groups_discover) |
| parser_groups_discover.add_argument( |
| "src_collection_dir", |
| type=pathlib.Path, |
| help="Path to raw collection directory where we will learn the " |
| "participant groups from", |
| ) |
| parser_groups_discover.add_argument( |
| "user_groups_csv", |
| type=pathlib.Path, |
| default="User_groups.csv", |
| help="The path to the User_groups.csv we will write to", |
| ) |
| |
| args = parser.parse_args(argv) |
| |
| if args.subcommand == "report": |
| if args.user_groups_csv and not args.user_groups_csv.is_file(): |
| parser.error("user-groups-csv must be a CSV file") |
| if not args.testcases_decisions_dir.is_dir(): |
| parser.error("testcases_decisions_dir must be a directory") |
| if args.analysis_dir.exists() and not args.analysis_dir.is_dir(): |
| parser.error("analysis_dir must be a directory") |
| elif args.subcommand == "groups-discover": |
| if not args.src_collection_dir.is_dir(): |
| parser.error("src_collection_dir must be a directory") |
| if args.user_groups_csv.exists(): |
| parser.error( |
| f"user_groups_csv {args.user_groups_csv} already exists" |
| ) |
| |
| return args.func(args) |
| |
| |
| if __name__ == "__main__": |
| sys.exit(main(sys.argv[1:])) |
