From a838c154135b2420d9047a101caf24a2c9f593c2 Mon Sep 17 00:00:00 2001 From: Scott Gasch Date: Sun, 11 Jul 2021 10:16:07 -0700 Subject: Random cleanups and type safety. Created ml subdir. --- acl.py | 6 +- config.py | 2 +- conversion_utils.py | 14 +- datetime_utils.py | 3 - dict_utils.py | 4 +- logging_utils.py | 4 +- ml/model_trainer.py | 393 ++++++++++++++++++++++++++++++++++++++++++++++ ml/quick_label.py | 123 +++++++++++++++ ml_model_trainer.py | 393 ---------------------------------------------- ml_quick_label.py | 123 --------------- string_utils.py | 8 +- tests/parallelize_test.py | 2 +- tests/run_all_tests.sh | 2 +- 13 files changed, 540 insertions(+), 537 deletions(-) create mode 100644 ml/model_trainer.py create mode 100644 ml/quick_label.py delete mode 100644 ml_model_trainer.py delete mode 100644 ml_quick_label.py diff --git a/acl.py b/acl.py index e6bb903..9155090 100644 --- a/acl.py +++ b/acl.py @@ -5,7 +5,7 @@ import enum import fnmatch import logging import re -from typing import Any, Callable, List, Optional, Set +from typing import Any, Callable, List, Optional, Set, Sequence # This module is commonly used by others in here and should avoid # taking any unnecessary dependencies back on them. @@ -134,8 +134,8 @@ class PredicateListBasedACL(SimpleACL): """An ACL that allows or denies by applying predicates.""" def __init__(self, *, - allow_predicate_list: List[Callable[[Any], bool]] = None, - deny_predicate_list: List[Callable[[Any], bool]] = None, + allow_predicate_list: Sequence[Callable[[Any], bool]] = None, + deny_predicate_list: Sequence[Callable[[Any], bool]] = None, order_to_check_allow_deny: Order, default_answer: bool) -> None: super().__init__( diff --git a/config.py b/config.py index e7094f3..672e1ae 100644 --- a/config.py +++ b/config.py @@ -161,7 +161,7 @@ def parse() -> Dict[str, Any]: """Main program should call this early in main()""" global config_parse_called if config_parse_called: - return + return config config_parse_called = True global saved_messages diff --git a/conversion_utils.py b/conversion_utils.py index 10908c5..b83d62e 100644 --- a/conversion_utils.py +++ b/conversion_utils.py @@ -15,15 +15,15 @@ class Converter(object): unit: str) -> None: self.name = name self.category = category - self.to_canonical = to_canonical - self.from_canonical = from_canonical + self.to_canonical_f = to_canonical + self.from_canonical_f = from_canonical self.unit = unit def to_canonical(self, n: Number) -> Number: - return self.to_canonical(n) + return self.to_canonical_f(n) def from_canonical(self, n: Number) -> Number: - return self.from_canonical(n) + return self.from_canonical_f(n) def unit_suffix(self) -> str: return self.unit @@ -75,7 +75,7 @@ conversion_catalog = { def convert(magnitude: Number, from_thing: str, - to_thing: str) -> Number: + to_thing: str) -> float: src = conversion_catalog.get(from_thing, None) dst = conversion_catalog.get(to_thing, None) if src is None or dst is None: @@ -87,10 +87,10 @@ def convert(magnitude: Number, def _convert(magnitude: Number, from_unit: Converter, - to_unit: Converter) -> Number: + to_unit: Converter) -> float: canonical = from_unit.to_canonical(magnitude) converted = to_unit.from_canonical(canonical) - return converted + return float(converted) def sec_to_min(s: float) -> float: diff --git a/datetime_utils.py b/datetime_utils.py index 795b427..f2cae8b 100644 --- a/datetime_utils.py +++ b/datetime_utils.py @@ -80,13 +80,10 @@ class TimeUnit(enum.Enum): @classmethod def is_valid(cls, value: Any): if type(value) is int: - print("int") return value in cls._value2member_map_ elif type(value) is TimeUnit: - print("TimeUnit") return value.value in cls._value2member_map_ elif type(value) is str: - print("str") return value in cls._member_names_ else: print(type(value)) diff --git a/dict_utils.py b/dict_utils.py index 0a2df25..292b933 100644 --- a/dict_utils.py +++ b/dict_utils.py @@ -39,9 +39,9 @@ def raise_on_duplicated_keys(key, v1, v2): def coalesce( inputs: Iterator[Dict[Any, Any]], *, - aggregation_function: Callable[[Any, Any, Any], Any] = coalesce_by_creating_list + aggregation_function: Callable[[Any, Any], Any] = coalesce_by_creating_list ) -> Dict[Any, Any]: - out = {} + out: Dict[Any, Any] = {} for d in inputs: for key in d: if key in out: diff --git a/logging_utils.py b/logging_utils.py index 700bfab..9c78f3f 100644 --- a/logging_utils.py +++ b/logging_utils.py @@ -102,7 +102,9 @@ class MillisecondAwareFormatter(logging.Formatter): converter = datetime.datetime.fromtimestamp def formatTime(self, record, datefmt=None): - ct = self.converter(record.created, pytz.timezone("US/Pacific")) + ct = MillisecondAwareFormatter.converter( + record.created, pytz.timezone("US/Pacific") + ) if datefmt: s = ct.strftime(datefmt) else: diff --git a/ml/model_trainer.py b/ml/model_trainer.py new file mode 100644 index 0000000..ab3059f --- /dev/null +++ b/ml/model_trainer.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from abc import ABC, abstractmethod +import datetime +import glob +import logging +import os +import pickle +import random +import sys +from types import SimpleNamespace +from typing import Any, List, NamedTuple, Optional, Set, Tuple + +import numpy as np +from sklearn.model_selection import train_test_split # type:ignore +from sklearn.preprocessing import MinMaxScaler # type: ignore + +from ansi import bold, reset +import argparse_utils +import config +from decorator_utils import timed +import parallelize as par + +logger = logging.getLogger(__file__) + +parser = config.add_commandline_args( + f"ML Model Trainer ({__file__})", + "Arguments related to training an ML model" +) +parser.add_argument( + "--ml_trainer_quiet", + action="store_true", + help="Don't prompt the user for anything." +) +parser.add_argument( + "--ml_trainer_delete", + action="store_true", + help="Delete invalid/incomplete features files in addition to warning." +) +group = parser.add_mutually_exclusive_group() +group.add_argument( + "--ml_trainer_dry_run", + action="store_true", + help="Do not write a new model, just report efficacy.", +) +group.add_argument( + "--ml_trainer_persist_threshold", + type=argparse_utils.valid_percentage, + metavar='0..100', + help="Persist the model if the test set score is >= this threshold.", +) + + +class InputSpec(SimpleNamespace): + file_glob: str + feature_count: int + features_to_skip: Set[str] + key_value_delimiter: str + training_parameters: List + label: str + basename: str + dry_run: Optional[bool] + quiet: Optional[bool] + persist_percentage_threshold: Optional[float] + delete_bad_inputs: Optional[bool] + + @staticmethod + def populate_from_config() -> InputSpec: + return InputSpec( + dry_run = config.config["ml_trainer_dry_run"], + quiet = config.config["ml_trainer_quiet"], + persist_percentage_threshold = config.config["ml_trainer_persist_threshold"], + delete_bad_inputs = config.config["ml_trainer_delete"], + ) + + +class OutputSpec(NamedTuple): + model_filename: Optional[str] + model_info_filename: Optional[str] + scaler_filename: Optional[str] + training_score: float + test_score: float + + +class TrainingBlueprint(ABC): + def __init__(self): + self.y_train = None + self.y_test = None + self.X_test_scaled = None + self.X_train_scaled = None + self.file_done_count = 0 + self.total_file_count = 0 + self.spec = None + + def train(self, spec: InputSpec) -> OutputSpec: + import smart_future + + random.seed() + self.spec = spec + + X_, y_ = self.read_input_files() + num_examples = len(y_) + + # Every example's features + X = np.array(X_) + + # Every example's label + y = np.array(y_) + + print("Doing random test/train split...") + X_train, X_test, self.y_train, self.y_test = self.test_train_split( + X, + y, + ) + + print("Scaling training data...") + scaler, self.X_train_scaled, self.X_test_scaled = self.scale_data( + X_train, + X_test, + ) + + print("Training model(s)...") + models = [] + modelid_to_params = {} + for params in self.spec.training_parameters: + model = self.train_model( + params, + self.X_train_scaled, + self.y_train + ) + models.append(model) + modelid_to_params[model.get_id()] = str(params) + + best_model = None + best_score = None + best_test_score = None + best_training_score = None + best_params = None + for model in smart_future.wait_any(models): + params = modelid_to_params[model.get_id()] + if isinstance(model, smart_future.SmartFuture): + model = model._resolve() + if model is not None: + training_score, test_score = self.evaluate_model( + model, + self.X_train_scaled, + self.y_train, + self.X_test_scaled, + self.y_test, + ) + score = (training_score + test_score * 20) / 21 + if not self.spec.quiet: + print( + f"{bold()}{params}{reset()}: " + f"Training set score={training_score:.2f}%, " + f"test set score={test_score:.2f}%", + file=sys.stderr, + ) + if best_score is None or score > best_score: + best_score = score + best_test_score = test_score + best_training_score = training_score + best_model = model + best_params = params + if not self.spec.quiet: + print( + f"New best score {best_score:.2f}% with params {params}" + ) + + if not self.spec.quiet: + msg = f"Done training; best test set score was: {best_test_score:.1f}%" + print(msg) + logger.info(msg) + scaler_filename, model_filename, model_info_filename = ( + self.maybe_persist_scaler_and_model( + best_training_score, + best_test_score, + best_params, + num_examples, + scaler, + best_model, + ) + ) + return OutputSpec( + model_filename = model_filename, + model_info_filename = model_info_filename, + scaler_filename = scaler_filename, + training_score = best_training_score, + test_score = best_test_score, + ) + + @par.parallelize(method=par.Method.THREAD) + def read_files_from_list( + self, + files: List[str], + n: int + ) -> Tuple[List, List]: + # All features + X = [] + + # The label + y = [] + + for filename in files: + wrote_label = False + with open(filename, "r") as f: + lines = f.readlines() + + # This example's features + x = [] + for line in lines: + + # We expect lines in features files to be of the form: + # + # key: value + line = line.strip() + try: + (key, value) = line.split(self.spec.key_value_delimiter) + except Exception as e: + logger.debug(f"WARNING: bad line in file {filename} '{line}', skipped") + continue + + key = key.strip() + value = value.strip() + if (self.spec.features_to_skip is not None + and key in self.spec.features_to_skip): + logger.debug(f"Skipping feature {key}") + continue + + value = self.normalize_feature(value) + + if key == self.spec.label: + y.append(value) + wrote_label = True + else: + x.append(value) + + # Make sure we saw a label and the requisite number of features. + if len(x) == self.spec.feature_count and wrote_label: + X.append(x) + self.file_done_count += 1 + else: + if wrote_label: + y.pop() + + if self.spec.delete_bad_inputs: + msg = f"WARNING: {filename}: missing features or label. DELETING." + print(msg, file=sys.stderr) + logger.warning(msg) + os.remove(filename) + else: + msg = f"WARNING: {filename}: missing features or label. Skipped." + print(msg, file=sys.stderr) + logger.warning(msg) + return (X, y) + + def make_progress_graph(self) -> None: + if not self.spec.quiet: + from text_utils import progress_graph + progress_graph( + self.file_done_count, + self.total_file_count + ) + + @timed + def read_input_files(self): + import list_utils + import smart_future + + # All features + X = [] + + # The label + y = [] + + results = [] + all_files = glob.glob(self.spec.file_glob) + self.total_file_count = len(all_files) + for n, files in enumerate(list_utils.shard(all_files, 500)): + file_list = list(files) + results.append(self.read_files_from_list(file_list, n)) + + for result in smart_future.wait_any(results, callback=self.make_progress_graph): + result = result._resolve() + for z in result[0]: + X.append(z) + for z in result[1]: + y.append(z) + if not self.spec.quiet: + print(" " * 80 + "\n") + return (X, y) + + def normalize_feature(self, value: str) -> Any: + if value in ("False", "None"): + ret = 0 + elif value == "True": + ret = 255 + elif isinstance(value, str) and "." in value: + ret = round(float(value) * 100.0) + else: + ret = int(value) + return ret + + def test_train_split(self, X, y) -> List: + logger.debug("Performing test/train split") + return train_test_split( + X, + y, + random_state=random.randrange(0, 1000), + ) + + def scale_data(self, + X_train: np.ndarray, + X_test: np.ndarray) -> Tuple[Any, np.ndarray, np.ndarray]: + logger.debug("Scaling data") + scaler = MinMaxScaler() + scaler.fit(X_train) + return (scaler, scaler.transform(X_train), scaler.transform(X_test)) + + # Note: children should implement. Consider using @parallelize. + @abstractmethod + def train_model(self, + parameters, + X_train_scaled: np.ndarray, + y_train: np.ndarray) -> Any: + pass + + def evaluate_model( + self, + model: Any, + X_train_scaled: np.ndarray, + y_train: np.ndarray, + X_test_scaled: np.ndarray, + y_test: np.ndarray) -> Tuple[np.float64, np.float64]: + logger.debug("Evaluating the model") + training_score = model.score(X_train_scaled, y_train) * 100.0 + test_score = model.score(X_test_scaled, y_test) * 100.0 + logger.info( + f"Model evaluation results: test_score={test_score:.5f}, " + f"train_score={training_score:.5f}" + ) + return (training_score, test_score) + + def maybe_persist_scaler_and_model( + self, + training_score: np.float64, + test_score: np.float64, + params: str, + num_examples: int, + scaler: Any, + model: Any) -> Tuple[Optional[str], Optional[str], Optional[str]]: + if not self.spec.dry_run: + import datetime_utils + import input_utils + import string_utils + + if ( + (self.spec.persist_percentage_threshold is not None and + test_score > self.spec.persist_percentage_threshold) + or + (not self.spec.quiet + and input_utils.yn_response("Write the model? [y,n]: ") == "y") + ): + scaler_filename = f"{self.spec.basename}_scaler.sav" + with open(scaler_filename, "wb") as f: + pickle.dump(scaler, f) + msg = f"Wrote {scaler_filename}" + print(msg) + logger.info(msg) + model_filename = f"{self.spec.basename}_model.sav" + with open(model_filename, "wb") as f: + pickle.dump(model, f) + msg = f"Wrote {model_filename}" + print(msg) + logger.info(msg) + model_info_filename = f"{self.spec.basename}_model_info.txt" + now: datetime.datetime = datetime_utils.now_pst() + info = f"""Timestamp: {datetime_utils.datetime_to_string(now)} +Model params: {params} +Training examples: {num_examples} +Training set score: {training_score:.2f}% +Testing set score: {test_score:.2f}%""" + with open(model_info_filename, "w") as f: + f.write(info) + msg = f"Wrote {model_info_filename}:" + print(msg) + logger.info(msg) + print(string_utils.indent(info, 2)) + logger.info(info) + return (scaler_filename, model_filename, model_info_filename) + return (None, None, None) diff --git a/ml/quick_label.py b/ml/quick_label.py new file mode 100644 index 0000000..1ed4296 --- /dev/null +++ b/ml/quick_label.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +import glob +import logging +import os +from typing import Callable, List, NamedTuple, Optional, Set + +import argparse_utils +import config + +logger = logging.getLogger(__name__) +parser = config.add_commandline_args( + f"ML Quick Labeler ({__file__})", + "Args related to quick labeling of ML training data", +) +parser.add_argument( + "--ml_quick_label_skip_list_path", + default="./qlabel_skip_list.txt", + metavar="FILENAME", + type=argparse_utils.valid_filename, + help="Path to file in which to store already labeled data.", +) +parser.add_argument( + "--ml_quick_label_use_skip_lists", + default=True, + action=argparse_utils.ActionNoYes, + help='Should we use a skip list file to speed up execution?', +) +parser.add_argument( + "--ml_quick_label_overwrite_labels", + default=False, + action=argparse_utils.ActionNoYes, + help='Enable overwriting existing labels; default is to not relabel.', +) + + +class InputSpec(NamedTuple): + image_file_glob: Optional[str] + image_file_prepopulated_list: Optional[List[str]] + image_file_to_features_file: Callable[[str], str] + label: str + valid_keystrokes: List[str] + prompt: str + keystroke_to_label: Callable[[str], str] + + +def read_skip_list() -> Set[str]: + ret: Set[str] = set() + if config.config['ml_quick_label_use_skip_lists']: + quick_skip_file = config.config['ml_quick_label_skip_list_path'] + if os.path.exists(quick_skip_file): + with open(quick_skip_file, 'r') as f: + lines = f.readlines() + for line in lines: + line = line[:-1] + line.strip() + ret.add(line) + logger.debug(f'Read {quick_skip_file} and found {len(ret)} entries.') + return ret + + +def write_skip_list(skip_list) -> None: + if config.config['ml_quick_label_use_skip_lists']: + quick_skip_file = config.config['ml_quick_label_skip_list_path'] + with open(quick_skip_file, 'w') as f: + for filename in skip_list: + filename = filename.strip() + if len(filename) > 0: + f.write(f'{filename}\n') + logger.debug(f'Updated {quick_skip_file}') + + +def label(in_spec: InputSpec) -> None: + import input_utils + + images = [] + if in_spec.image_file_glob is not None: + images += glob.glob(in_spec.image_file_glob) + elif in_spec.image_file_prepopulated_list is not None: + images += in_spec.image_file_prepopulated_list + else: + raise ValueError( + 'One of image_file_glob or image_file_prepopulated_list is required' + ) + + skip_list = read_skip_list() + for image in images: + if image in skip_list: + logger.debug(f'Skipping {image} because of the skip list') + continue + features = in_spec.image_file_to_features_file(image) + if features is None or not os.path.exists(features): + logger.warning( + f'File {image} yielded file {features} which does not exist, SKIPPING.' + ) + continue + + # Render features and image. + filtered_lines = [] + with open(features, "r") as f: + lines = f.readlines() + saw_label = False + for line in lines: + line = line[:-1] + if in_spec.label not in line: + filtered_lines.append(line) + else: + saw_label = True + + if not saw_label or config.config['ml_quick_label_overwrite_labels']: + logger.info(features) + os.system(f'xv {image} &') + keystroke = input_utils.single_keystroke_response( + in_spec.valid_keystrokes, + prompt=in_spec.prompt, + ) + os.system('killall xv') + label_value = in_spec.keystroke_to_label(keystroke) + filtered_lines.append(f"{in_spec.label}: {label_value}\n") + with open(features, 'w') as f: + f.writelines("%s\n" % line for line in filtered_lines) + skip_list.add(image) + write_skip_list(skip_list) diff --git a/ml_model_trainer.py b/ml_model_trainer.py deleted file mode 100644 index ab3059f..0000000 --- a/ml_model_trainer.py +++ /dev/null @@ -1,393 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -from abc import ABC, abstractmethod -import datetime -import glob -import logging -import os -import pickle -import random -import sys -from types import SimpleNamespace -from typing import Any, List, NamedTuple, Optional, Set, Tuple - -import numpy as np -from sklearn.model_selection import train_test_split # type:ignore -from sklearn.preprocessing import MinMaxScaler # type: ignore - -from ansi import bold, reset -import argparse_utils -import config -from decorator_utils import timed -import parallelize as par - -logger = logging.getLogger(__file__) - -parser = config.add_commandline_args( - f"ML Model Trainer ({__file__})", - "Arguments related to training an ML model" -) -parser.add_argument( - "--ml_trainer_quiet", - action="store_true", - help="Don't prompt the user for anything." -) -parser.add_argument( - "--ml_trainer_delete", - action="store_true", - help="Delete invalid/incomplete features files in addition to warning." -) -group = parser.add_mutually_exclusive_group() -group.add_argument( - "--ml_trainer_dry_run", - action="store_true", - help="Do not write a new model, just report efficacy.", -) -group.add_argument( - "--ml_trainer_persist_threshold", - type=argparse_utils.valid_percentage, - metavar='0..100', - help="Persist the model if the test set score is >= this threshold.", -) - - -class InputSpec(SimpleNamespace): - file_glob: str - feature_count: int - features_to_skip: Set[str] - key_value_delimiter: str - training_parameters: List - label: str - basename: str - dry_run: Optional[bool] - quiet: Optional[bool] - persist_percentage_threshold: Optional[float] - delete_bad_inputs: Optional[bool] - - @staticmethod - def populate_from_config() -> InputSpec: - return InputSpec( - dry_run = config.config["ml_trainer_dry_run"], - quiet = config.config["ml_trainer_quiet"], - persist_percentage_threshold = config.config["ml_trainer_persist_threshold"], - delete_bad_inputs = config.config["ml_trainer_delete"], - ) - - -class OutputSpec(NamedTuple): - model_filename: Optional[str] - model_info_filename: Optional[str] - scaler_filename: Optional[str] - training_score: float - test_score: float - - -class TrainingBlueprint(ABC): - def __init__(self): - self.y_train = None - self.y_test = None - self.X_test_scaled = None - self.X_train_scaled = None - self.file_done_count = 0 - self.total_file_count = 0 - self.spec = None - - def train(self, spec: InputSpec) -> OutputSpec: - import smart_future - - random.seed() - self.spec = spec - - X_, y_ = self.read_input_files() - num_examples = len(y_) - - # Every example's features - X = np.array(X_) - - # Every example's label - y = np.array(y_) - - print("Doing random test/train split...") - X_train, X_test, self.y_train, self.y_test = self.test_train_split( - X, - y, - ) - - print("Scaling training data...") - scaler, self.X_train_scaled, self.X_test_scaled = self.scale_data( - X_train, - X_test, - ) - - print("Training model(s)...") - models = [] - modelid_to_params = {} - for params in self.spec.training_parameters: - model = self.train_model( - params, - self.X_train_scaled, - self.y_train - ) - models.append(model) - modelid_to_params[model.get_id()] = str(params) - - best_model = None - best_score = None - best_test_score = None - best_training_score = None - best_params = None - for model in smart_future.wait_any(models): - params = modelid_to_params[model.get_id()] - if isinstance(model, smart_future.SmartFuture): - model = model._resolve() - if model is not None: - training_score, test_score = self.evaluate_model( - model, - self.X_train_scaled, - self.y_train, - self.X_test_scaled, - self.y_test, - ) - score = (training_score + test_score * 20) / 21 - if not self.spec.quiet: - print( - f"{bold()}{params}{reset()}: " - f"Training set score={training_score:.2f}%, " - f"test set score={test_score:.2f}%", - file=sys.stderr, - ) - if best_score is None or score > best_score: - best_score = score - best_test_score = test_score - best_training_score = training_score - best_model = model - best_params = params - if not self.spec.quiet: - print( - f"New best score {best_score:.2f}% with params {params}" - ) - - if not self.spec.quiet: - msg = f"Done training; best test set score was: {best_test_score:.1f}%" - print(msg) - logger.info(msg) - scaler_filename, model_filename, model_info_filename = ( - self.maybe_persist_scaler_and_model( - best_training_score, - best_test_score, - best_params, - num_examples, - scaler, - best_model, - ) - ) - return OutputSpec( - model_filename = model_filename, - model_info_filename = model_info_filename, - scaler_filename = scaler_filename, - training_score = best_training_score, - test_score = best_test_score, - ) - - @par.parallelize(method=par.Method.THREAD) - def read_files_from_list( - self, - files: List[str], - n: int - ) -> Tuple[List, List]: - # All features - X = [] - - # The label - y = [] - - for filename in files: - wrote_label = False - with open(filename, "r") as f: - lines = f.readlines() - - # This example's features - x = [] - for line in lines: - - # We expect lines in features files to be of the form: - # - # key: value - line = line.strip() - try: - (key, value) = line.split(self.spec.key_value_delimiter) - except Exception as e: - logger.debug(f"WARNING: bad line in file {filename} '{line}', skipped") - continue - - key = key.strip() - value = value.strip() - if (self.spec.features_to_skip is not None - and key in self.spec.features_to_skip): - logger.debug(f"Skipping feature {key}") - continue - - value = self.normalize_feature(value) - - if key == self.spec.label: - y.append(value) - wrote_label = True - else: - x.append(value) - - # Make sure we saw a label and the requisite number of features. - if len(x) == self.spec.feature_count and wrote_label: - X.append(x) - self.file_done_count += 1 - else: - if wrote_label: - y.pop() - - if self.spec.delete_bad_inputs: - msg = f"WARNING: {filename}: missing features or label. DELETING." - print(msg, file=sys.stderr) - logger.warning(msg) - os.remove(filename) - else: - msg = f"WARNING: {filename}: missing features or label. Skipped." - print(msg, file=sys.stderr) - logger.warning(msg) - return (X, y) - - def make_progress_graph(self) -> None: - if not self.spec.quiet: - from text_utils import progress_graph - progress_graph( - self.file_done_count, - self.total_file_count - ) - - @timed - def read_input_files(self): - import list_utils - import smart_future - - # All features - X = [] - - # The label - y = [] - - results = [] - all_files = glob.glob(self.spec.file_glob) - self.total_file_count = len(all_files) - for n, files in enumerate(list_utils.shard(all_files, 500)): - file_list = list(files) - results.append(self.read_files_from_list(file_list, n)) - - for result in smart_future.wait_any(results, callback=self.make_progress_graph): - result = result._resolve() - for z in result[0]: - X.append(z) - for z in result[1]: - y.append(z) - if not self.spec.quiet: - print(" " * 80 + "\n") - return (X, y) - - def normalize_feature(self, value: str) -> Any: - if value in ("False", "None"): - ret = 0 - elif value == "True": - ret = 255 - elif isinstance(value, str) and "." in value: - ret = round(float(value) * 100.0) - else: - ret = int(value) - return ret - - def test_train_split(self, X, y) -> List: - logger.debug("Performing test/train split") - return train_test_split( - X, - y, - random_state=random.randrange(0, 1000), - ) - - def scale_data(self, - X_train: np.ndarray, - X_test: np.ndarray) -> Tuple[Any, np.ndarray, np.ndarray]: - logger.debug("Scaling data") - scaler = MinMaxScaler() - scaler.fit(X_train) - return (scaler, scaler.transform(X_train), scaler.transform(X_test)) - - # Note: children should implement. Consider using @parallelize. - @abstractmethod - def train_model(self, - parameters, - X_train_scaled: np.ndarray, - y_train: np.ndarray) -> Any: - pass - - def evaluate_model( - self, - model: Any, - X_train_scaled: np.ndarray, - y_train: np.ndarray, - X_test_scaled: np.ndarray, - y_test: np.ndarray) -> Tuple[np.float64, np.float64]: - logger.debug("Evaluating the model") - training_score = model.score(X_train_scaled, y_train) * 100.0 - test_score = model.score(X_test_scaled, y_test) * 100.0 - logger.info( - f"Model evaluation results: test_score={test_score:.5f}, " - f"train_score={training_score:.5f}" - ) - return (training_score, test_score) - - def maybe_persist_scaler_and_model( - self, - training_score: np.float64, - test_score: np.float64, - params: str, - num_examples: int, - scaler: Any, - model: Any) -> Tuple[Optional[str], Optional[str], Optional[str]]: - if not self.spec.dry_run: - import datetime_utils - import input_utils - import string_utils - - if ( - (self.spec.persist_percentage_threshold is not None and - test_score > self.spec.persist_percentage_threshold) - or - (not self.spec.quiet - and input_utils.yn_response("Write the model? [y,n]: ") == "y") - ): - scaler_filename = f"{self.spec.basename}_scaler.sav" - with open(scaler_filename, "wb") as f: - pickle.dump(scaler, f) - msg = f"Wrote {scaler_filename}" - print(msg) - logger.info(msg) - model_filename = f"{self.spec.basename}_model.sav" - with open(model_filename, "wb") as f: - pickle.dump(model, f) - msg = f"Wrote {model_filename}" - print(msg) - logger.info(msg) - model_info_filename = f"{self.spec.basename}_model_info.txt" - now: datetime.datetime = datetime_utils.now_pst() - info = f"""Timestamp: {datetime_utils.datetime_to_string(now)} -Model params: {params} -Training examples: {num_examples} -Training set score: {training_score:.2f}% -Testing set score: {test_score:.2f}%""" - with open(model_info_filename, "w") as f: - f.write(info) - msg = f"Wrote {model_info_filename}:" - print(msg) - logger.info(msg) - print(string_utils.indent(info, 2)) - logger.info(info) - return (scaler_filename, model_filename, model_info_filename) - return (None, None, None) diff --git a/ml_quick_label.py b/ml_quick_label.py deleted file mode 100644 index 1ed4296..0000000 --- a/ml_quick_label.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python3 - -import glob -import logging -import os -from typing import Callable, List, NamedTuple, Optional, Set - -import argparse_utils -import config - -logger = logging.getLogger(__name__) -parser = config.add_commandline_args( - f"ML Quick Labeler ({__file__})", - "Args related to quick labeling of ML training data", -) -parser.add_argument( - "--ml_quick_label_skip_list_path", - default="./qlabel_skip_list.txt", - metavar="FILENAME", - type=argparse_utils.valid_filename, - help="Path to file in which to store already labeled data.", -) -parser.add_argument( - "--ml_quick_label_use_skip_lists", - default=True, - action=argparse_utils.ActionNoYes, - help='Should we use a skip list file to speed up execution?', -) -parser.add_argument( - "--ml_quick_label_overwrite_labels", - default=False, - action=argparse_utils.ActionNoYes, - help='Enable overwriting existing labels; default is to not relabel.', -) - - -class InputSpec(NamedTuple): - image_file_glob: Optional[str] - image_file_prepopulated_list: Optional[List[str]] - image_file_to_features_file: Callable[[str], str] - label: str - valid_keystrokes: List[str] - prompt: str - keystroke_to_label: Callable[[str], str] - - -def read_skip_list() -> Set[str]: - ret: Set[str] = set() - if config.config['ml_quick_label_use_skip_lists']: - quick_skip_file = config.config['ml_quick_label_skip_list_path'] - if os.path.exists(quick_skip_file): - with open(quick_skip_file, 'r') as f: - lines = f.readlines() - for line in lines: - line = line[:-1] - line.strip() - ret.add(line) - logger.debug(f'Read {quick_skip_file} and found {len(ret)} entries.') - return ret - - -def write_skip_list(skip_list) -> None: - if config.config['ml_quick_label_use_skip_lists']: - quick_skip_file = config.config['ml_quick_label_skip_list_path'] - with open(quick_skip_file, 'w') as f: - for filename in skip_list: - filename = filename.strip() - if len(filename) > 0: - f.write(f'{filename}\n') - logger.debug(f'Updated {quick_skip_file}') - - -def label(in_spec: InputSpec) -> None: - import input_utils - - images = [] - if in_spec.image_file_glob is not None: - images += glob.glob(in_spec.image_file_glob) - elif in_spec.image_file_prepopulated_list is not None: - images += in_spec.image_file_prepopulated_list - else: - raise ValueError( - 'One of image_file_glob or image_file_prepopulated_list is required' - ) - - skip_list = read_skip_list() - for image in images: - if image in skip_list: - logger.debug(f'Skipping {image} because of the skip list') - continue - features = in_spec.image_file_to_features_file(image) - if features is None or not os.path.exists(features): - logger.warning( - f'File {image} yielded file {features} which does not exist, SKIPPING.' - ) - continue - - # Render features and image. - filtered_lines = [] - with open(features, "r") as f: - lines = f.readlines() - saw_label = False - for line in lines: - line = line[:-1] - if in_spec.label not in line: - filtered_lines.append(line) - else: - saw_label = True - - if not saw_label or config.config['ml_quick_label_overwrite_labels']: - logger.info(features) - os.system(f'xv {image} &') - keystroke = input_utils.single_keystroke_response( - in_spec.valid_keystrokes, - prompt=in_spec.prompt, - ) - os.system('killall xv') - label_value = in_spec.keystroke_to_label(keystroke) - filtered_lines.append(f"{in_spec.label}: {label_value}\n") - with open(features, 'w') as f: - f.writelines("%s\n" % line for line in filtered_lines) - skip_list.add(image) - write_skip_list(skip_list) diff --git a/string_utils.py b/string_utils.py index 740a0b9..911008d 100644 --- a/string_utils.py +++ b/string_utils.py @@ -225,10 +225,14 @@ def strip_escape_sequences(in_str: str) -> str: return in_str -def add_thousands_separator(in_str: str, *, separator_char = ',', places = 3) -> str: +def add_thousands_separator( + in_str: str, + *, + separator_char = ',', + places = 3 +) -> str: if isinstance(in_str, int): in_str = f'{in_str}' - if is_number(in_str): return _add_thousands_separator( in_str, diff --git a/tests/parallelize_test.py b/tests/parallelize_test.py index 051ec5d..44f723c 100755 --- a/tests/parallelize_test.py +++ b/tests/parallelize_test.py @@ -22,7 +22,7 @@ def list_primes(n): @decorator_utils.timed def driver() -> None: results = {} - for _ in range(200): + for _ in range(50): n = random.randint(0, 100000) results[n] = list_primes(n) tot = 0 diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index ecb3648..13aa2fb 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -1,6 +1,6 @@ #!/bin/bash for test in $(ls *_test.py); do - echo "------------------------------ ${test} ------------------------------" + echo "------------------------- ${test} -------------------------" ${test} done -- cgit v1.3