#!/usr/bin/env python3 """ espace_citoyens_cli =================== A small command‑line utility that logs into a French “espace‑citoyens” municipal portal, lists the different reservation categories that are available for a given city and, on request, exports the reservations of a chosen category to an iCalendar (``.ics``) file. The script is deliberately *fail‑fast*: any unrecoverable error (missing configuration, HTTP error, unexpected HTML markup, unknown reservation code, etc.) is logged and the process terminates with ``sys.exit(-1)``. This makes failures immediately visible in CI pipelines or in automated scripts that call the tool. Why the script exists --------------------- Many French municipalities expose a public “espace‑citoyens” where residents can book facilities (sports halls, cultural rooms, playgrounds, ...). The web interface is JavaScript‑driven and does not provide an official API. This module reproduces the steps a browser would take, extracts the reservation data, and converts it into a standard iCalendar file that can be imported into any calendar application (Outlook, Google Calendar, Apple Calendar, ...). High‑level workflow ------------------- 1. **Authentication** – ``authenticate()`` fetches the public landing page, discovers the login form, extracts the hidden ``__RequestVerificationToken``, posts the credentials, follows the 302 redirect and returns the URL of the user’s main page together with the session cookies. 2. **Discover reservation kinds** – ``get_reservations_kind()`` loads the main page, parses the *tblDalleDetail_Reservation* table and builds a mapping ``{kind_name: absolute_url}``. 3. **Export a reservation calendar** – ``dump_reservation()`` retrieves the JavaScript variables required for the calendar request, calls the JSON‑based calendar endpoint, walks the nested week → day → reservation structure, builds an ``ics.Calendar`` object and writes it to the requested output file. 4. **Command‑line entry point** – ``main()`` parses arguments (login, city, optional config file, debug flag, sub‑command), merges a possible configuration file, performs authentication and dispatches to the functions above. Public API ---------- - :func:`get_key(dictionnary, key) → Any` - :func:`authenticate(base_url, city, login, password) → (str, RequestsCookieJar)` - :func:`get_reservations_kind(base_url, url, cookies) → dict[str, str]` - :func:`dump_reservation(base_url, city, url, cookies, output) → None` - :func:`main() → None` Typical usage ------------- ```bash # 1. Verify that the credentials work $ python reservation_cli.py -l john.doe@example.com -C paris A # 2. List all reservation categories for the city $ python reservation_cli.py -l john -C lyon L 1 - Sport 2 - Culture 3 - Bibliothèque # 3. Export the second category to an iCalendar file $ python reservation_cli.py -l john -C lyon D -i 2 -o my_reservations.ics """ import configparser import argparse import logging import json import re import sys from io import StringIO import getpass from zoneinfo import ZoneInfo from datetime import datetime from collections.abc import Hashable from typing import Any from requests.cookies import RequestsCookieJar from typeguard import typechecked import requests import coloredlogs from bs4 import BeautifulSoup from ics import Calendar, Event @typechecked def get_key(dictionnary: dict, key: Hashable) -> Any: """ Retrieve a value from a dictionary, logging an error and terminating the program if the requested key is missing. This helper centralises the “fail‑fast” pattern for required configuration values (or any other mandatory mapping entries). Instead of propagating a `KeyError`, the function logs a clear error message and exits with a non‑zero status code, making the failure immediately visible in logs and in the process exit code. Args: dictionnary (dict): The dictionary (or mapping) to search. The name *dictionnary* mirrors the original code, but any mapping type that supports ``__getitem__`` works (e.g., ``dict``, ``collections.UserDict``, ``defaultdict``). key (Hashable): The key whose associated value should be returned. Returns: Any: The value stored under ``key`` in ``dictionnary``. Raises: SystemExit: If ``key`` is not present in ``dictionnary``. The function logs an error message of the form ``"Missing key: "`` before exiting with status ``-1`` (equivalent to an exit code of ``255`` on most POSIX systems). Side Effects: * Writes an error entry to the logger for the current module (``logging.getLogger(__name__)``). * Calls ``sys.exit(-1)`` which terminates the interpreter. Example: >>> config = {"host": "localhost", "port": 8080} >>> get_key(config, "host") 'localhost' >>> get_key(config, "username") # Logs: "Missing key: username" # Terminates the program with exit code -1 """ logger = logging.getLogger(__name__) try: return dictionnary[key] except KeyError: logger.error('Missing key: %s', key) sys.exit(-1) @typechecked def authenticate(base_url: str, city: str, login: str, password: str) -> (str, RequestsCookieJar): """ Perform a two‑step web authentication for a citizen‑portal site and return the URL of the user’s main page together with the session cookies. The routine mimics the steps a browser would take: 1. Retrieve the public “espace‑citoyens” page for the specified city. 2. Parse the HTML to discover the form‑action URL used for login. 3. Extract the hidden ``__RequestVerificationToken`` required for the POST. 4. Submit the credentials and token via a POST request. 5. Verify that the server responded with a 302 redirect (successful login) and build the absolute URL of the landing page. All failures are logged (using the module‑level logger) and cause the process to exit with status ``-1`` via ``sys.exit``. Args: base_url (str): The root of the portal, e.g. ``"https://example.com"``. The function will append ``//espace-citoyens`` to form the initial URL. city (str): Identifier of the city (or sub‑site) to target, inserted into the URL path. login (str): The username/email used for authentication. password (str): The password associated with ``login``. Returns: tuple: ``(main_page_url, cookies)`` where: * **main_page_url** (str) – Absolute URL of the page the user is redirected to after a successful login (taken from the ``Location`` header of the 302 response). * **cookies** (RequestsCookieJar) – Cookie jar captured from the first GET request; it must be passed to subsequent requests to keep the session alive. Raises: SystemExit: The function deliberately exits the interpreter on any of the following conditions, after writing an appropriate error message to the logger: * Unable to fetch the initial page (non‑200 status). * Unable to locate the login form’s ``action`` URL. * Unable to extract the hidden ``__RequestVerificationToken``. * Login POST does not return a 302 redirect. Side Effects: * Writes informational, debug, and error messages to the module logger. * Calls ``sys.exit(-1)`` on fatal errors, terminating the process. * Performs network I/O via the ``requests`` library. Example: >>> base = "https://mycity.gov" >>> city = "paris" >>> user = "john.doe@example.com" >>> pwd = "s3cr3t" >>> main_url, sess_cookies = authenticate(base, city, user, pwd) >>> print(main_url) https://mycity.gov/dashboard >>> # Subsequent requests can reuse ``sess_cookies``: >>> resp = requests.get(main_url, cookies=sess_cookies) Note: The function parses the HTML with regular expressions and ``StringIO``; it assumes the login form and token follow the exact patterns used in the original site. If the page layout changes, the regexes may need to be updated. """ logger = logging.getLogger(__name__) url = f"{base_url}/{city}/espace-citoyens" logger.info('Retrieve base site') html = requests.get(url, allow_redirects=False, timeout=10) if html.status_code != 200: logger.error('Impossible to retrieve Web site: %d', html.status_code) sys.exit(-1) cookies = html.cookies found = False p = re.compile('^.*
dict[str, str]: """ Scrape a reservation‑portal page and return a mapping of reservation‑type names → absolute URLs that list the corresponding reservations. The function expects the *url* argument to point to a page that contains a single ```` element. Each row of the table represents a different kind of reservation (e.g. “Sport”, “Culture”, …). For each row the function extracts: * the textual description found in the ``
`` whose *id* matches the regular expression ``ReservationActivite``; * the relative hyperlink (``href``) of the first ```` tag in the row. The relative link is then prefixed with *base_url* to produce an absolute URL. All pairs are collected in a dictionary and returned to the caller. **Behaviour on failure** – The function logs the problem and terminates the process with ``sys.exit(-1)`` (i.e. a ``SystemExit`` exception). This mirrors the “fail‑fast” style used throughout the code base. Args: base_url (str): The root URL of the site (e.g. ``"https://example.com"`` without a trailing slash). It is concatenated with the relative links found in the table to build the final URLs. url (str): The full URL of the page that lists the different reservation types. cookies (requests.cookies.RequestsCookieJar): Cookie jar obtained during the authentication step; passed to ``requests.get`` so the request is made within the logged‑in session. Returns: dict[str, str]: A dictionary where each key is the reservation type name (as extracted from the table cell) and each value is the absolute URL that leads to the detailed list of reservations for that type. Raises: SystemExit: * If the HTTP GET request does not return a *200* status. * If the page does not contain exactly one table with the expected ``id`` (either none found or more than one). * Any other unexpected parsing error that would lead to a missing element (the code will log an error and exit). Side effects: * Writes informational and error messages to the module logger. * Calls ``sys.exit(-1)`` on fatal errors, terminating the interpreter. Example: >>> base = "https://mycity.gov" >>> page = "https://mycity.gov/paris/espace-citoyens/reservations" >>> session_cookies = authenticate(base, "paris", "john", "s3cr3t")[1] >>> kinds = get_reservations_kind(base, page, session_cookies) >>> for kind, link in kinds.items(): ... print(f"{kind}: {link}") Sport: https://mycity.gov/paris/reservations/sport Culture: https://mycity.gov/paris/reservations/culture Note: The function uses **BeautifulSoup** (HTML parser) and a simple regular expression to locate the ```` element that holds the reservation type. If the upstream website changes its markup (different table id, different column names, etc.) the function will need to be updated accordingly. """ logger = logging.getLogger(__name__) # Retrieve main page html = requests.get(url, cookies=cookies, allow_redirects=False, timeout=10) if html.status_code != 200: logger.error('Impossible to retrieve main page: %d', html.status_code) sys.exit(-1) soup = BeautifulSoup(html.text, 'html.parser') resas = soup.find_all('table', id='tblDalleDetail_Reservation') if len(resas) != 1: logger.error('Too many kind of reservations') sys.exit(-1) resas = resas[0] resas = resas.find_all('tr') resa_types = {} for resa in resas: resa_type = resa.find('td', id=re.compile('ReservationActivite')).get_text() url = resa.find('a').get('href') resa_types[resa_type] = base_url+url return resa_types def dump_reservation(base_url: str, city: str, url: str, cookies: RequestsCookieJar, output: str) -> None: """ Download a citizen‑portal reservation calendar, convert it to iCalendar format and write the result to *output*. The routine performs the full extraction pipeline that the original web interface uses: 1. **Fetch the reservation page** (``url``) and parse four hidden JavaScript variables – ``idPer``, ``idIns``, ``idLie`` and ``idClg`` – that are later required to request the calendar data. 2. **Query the calendar endpoint** ``/espace-citoyens/DemandeEnfance/ NouvelleDemandeReservationGetCalendrier`` with the extracted variables as GET parameters. 3. **Deserialize the JSON payload** and walk through the nested structure (weeks → days → reservation units) to build an ``ics.Calendar`` object. 4. **Derive start/end times** from the reservation code (e.g. “Matin”, “AM”, “Repas”). Times are expressed in the *Europe/Paris* time‑zone. 5. **Serialise the calendar** and write it as UTF‑8 text to *output*. The function follows the **fail‑fast** philosophy used throughout the project: any unexpected HTTP status code, missing variable, or unknown reservation code is logged as an error and the process terminates with ``sys.exit(-1)`` (i.e. a ``SystemExit`` exception). Parameters ---------- base_url : str Root URL of the portal, e.g. ``"https://mycity.gov"`` (no trailing slash). city : str City identifier that is part of the calendar endpoint’s path (e.g. ``"paris"``). url : str Full URL of the page that contains the JavaScript variables needed for the calendar request. cookies : requests.cookies.RequestsCookieJar Cookie jar obtained from the authentication step; it must be passed to every request to stay within the authenticated session. output : str Path of the file that will receive the iCalendar representation (the file is opened in *write* mode with UTF‑8 encoding). Returns ------- None The function writes its result to *output*; it does not return a value. Raises ------ SystemExit The function calls ``sys.exit(-1)`` (and logs an error) when encountering any of the following conditions: * HTTP GET to *url* or to the calendar endpoint returns a status code other than **200**. * One of the required JavaScript variables (`idPer`, `idIns`, `idLie`, `idClg`) cannot be found in the page source. * The JSON payload does not contain the expected keys (`listeSemainesAffichees`, `listeUnitesInscr`, …). * A reservation code cannot be mapped to a time slot. Side Effects ------------ * Writes log messages (INFO, DEBUG, WARNING, ERROR) to the module’s logger. * Terminates the interpreter on fatal errors via ``sys.exit``. * Creates and writes an iCalendar file on disk. Example ------- ```python # Assume the user is already authenticated and we have a cookie jar: base = "https://mycity.gov" city = "paris" page = f"{base}/{city}/espace-citoyens/reservations" cookies = authenticate(base, city, "john.doe@example.com", "s3cr3t")[1] dump_reservation( base_url=base, city=city, url=page, cookies=cookies, output="my_reservations.ics" ) # → ``my_reservations.ics`` now contains a standard iCalendar file that can # be imported into Outlook, Google Calendar, etc. ``` Notes ----- """ logger = logging.getLogger(__name__) html = requests.get(url, cookies=cookies, allow_redirects=False, timeout=10) if html.status_code != 200: logger.info('Impossible to retrieve reservation page: %d', html.status_code) sys.exit(-1) variables = ['idPer', 'idIns', 'idLie', 'idClg'] values = {} for var in variables: found = False content = StringIO(html.content.decode('utf8')) p = re.compile(f'^.*var {var} = (?P[0-9]+).*$') for line in content.readlines(): m = p.match(line) if m is not None: found = True value = int(m.group('value')) values[var] = value break if not found: logger.error('Impossible to find value for variable: %s', var) sys.exit(-1) else: logger.debug('Found value for var %s: %d', var, value) # This URL should be retrieved more automatically ... calendar = requests.get(f"{base_url}/{city}/\ espace-citoyens/DemandeEnfance/NouvelleDemandeReservationGetCalendrier", params=values, cookies=cookies, timeout=10) if calendar.status_code != 200: logger.info('Impossible to retrieve calendar: %d', html.status_code) sys.exit(-1) calendar = json.load(StringIO(calendar.content.decode('utf8'))) weeks = get_key(calendar, 'listeSemainesAffichees') type_resas = get_key(calendar, 'listeUnitesInscr') dict_resa = {} for type_resa in type_resas: id_resa = get_key(type_resa, 'idUnite') code_resa = get_key(type_resa, 'codeUnite') desc_resa = get_key(type_resa, 'libUnite') dict_resa[id_resa] = (desc_resa, code_resa) cal = Calendar() for week in weeks: num_semaine = int(get_key(week, 'numSemaine')) days = get_key(week, 'listeJoursAffiches') for day in days: resas = get_key(day, 'listeUnitesJour') date = int(get_key(day, 'idJour')) year = int(date/10000) month = int((date - year*10000) / 100) day = date - year*10000 - month*100 for resa in resas: checked = get_key(resa, 'nbConsoBase') is not None if checked: type_resa = get_key(resa, 'idUnite') e = Event() e.name = dict_resa[type_resa][0] code = dict_resa[type_resa][1] if 'Matin' in code: begin = datetime(year,month,day,7,30,0, tzinfo=ZoneInfo("Europe/Paris")) end = datetime(year,month,day,12,0,0, tzinfo=ZoneInfo("Europe/Paris")) elif 'AM' in code: begin = datetime(year,month,day,13,30,0, tzinfo=ZoneInfo("Europe/Paris")) end = datetime(year,month,day,18,0,0, tzinfo=ZoneInfo("Europe/Paris")) elif 'Repas' in code: begin = datetime(year,month,day,12,00,0, tzinfo=ZoneInfo("Europe/Paris")) end = datetime(year,month,day,13,30,0, tzinfo=ZoneInfo("Europe/Paris")) else: logger.warning('Impossible to determine the type of reservation: %s', code) begin = datetime(year,month,day) end = datetime(year,month,day) e.begin = begin e.end = end cal.events.add(e) with open(output, 'w', encoding="utf-8") as f: f.writelines(cal.serialize_iter()) def main(): """ Command‑line entry point that drives the whole *espace‑citoyens* reservation workflow. The script can be used in three modes, selected through sub‑commands: * ``A`` – Test the authentication step only. * ``L`` – List every available reservation type (e.g. “Sports”, “Culture”). * ``D`` – Export the reservations of a specific type to an iCalendar file. The function parses command‑line arguments, optionally merges them with a configuration file, performs authentication, and then dispatches to the appropriate helper functions (`authenticate`, `get_reservations_kind`, `dump_reservation`). All errors are logged and cause the process to exit with ``sys.exit(-1)`` (i.e. a non‑zero status code). No explicit parameters are passed to the function – it reads ``sys.argv`` directly via ``argparse`` – and it does not return a value; the program terminates either normally (``sys.exit(0)``) or with an error code. Workflow summary ---------------- 1. **Argument parsing** – Handles login, city, optional config file, password, debug flag and the three sub‑commands. The ``-d/--debug`` switch raises the logging level to ``DEBUG``. 2. **Configuration file** – If ``--config`` is supplied, the file is read with ``configparser``. Settings in the ``[Login]`` and ``[Espace]`` sections override the corresponding command‑line options. 3. **Mandatory values** – ``city`` and ``login`` must finally be known; otherwise the script prints the help text and exits with an error. 4. **Password handling** – If no password is provided via CLI or config, the user is prompted securely with ``getpass.getpass()``. 5. **Authentication** – Calls :func:`authenticate` to obtain the main page URL and the session cookies. 6. **Command dispatch** * ``A`` (or no sub‑command) → exit after a successful login. * ``L`` → list reservation types (printed to stdout). * ``D`` → export the reservations of the selected type: * validates the supplied index, * retrieves the URL for the chosen reservation kind, * calls :func:`dump_reservation` to create the ``.ics`` file. Parameters ---------- None (uses ``sys.argv``). Returns ------- None. The function terminates the process with ``sys.exit``. Raises ------ SystemExit * Invalid command‑line usage (handled by ``argparse``). * Failure to read or parse the optional configuration file. * Missing required arguments (city or login). * Authentication failure (non‑200 response from the portal). * Invalid reservation index for the ``D`` sub‑command. * Any other unrecoverable error reported by the helper functions. Side effects ------------ * Configures coloured logging via ``coloredlogs``. * Writes log messages (INFO, DEBUG, WARNING, ERROR) to the module logger. * May prompt the user for a password on the terminal. * Reads a configuration file if supplied. * Performs HTTP requests to the remote portal. * Writes an iCalendar file to disk when the ``D`` command is used. Example usage ------------- ```bash # 1. Test authentication only $ python reservation_cli.py -l john.doe@example.com -C paris A # 2. List all reservation categories $ python reservation_cli.py -l john -C lyon L 1 - Sport 2 - Culture 3 - Bibliothèque # 3. Export the second reservation type to a calendar file $ python reservation_cli.py -l john -C lyon D -i 2 -o my_reservations.ics ``` Notes ----- * The script relies on the third‑party libraries **coloredlogs**, **argparse**, **configparser**, **requests**, **BeautifulSoup**, **ics**, and the standard **logging**, **getpass**, **sys**, **json**, **datetime**, and **zoneinfo** modules. * All helper functions (`authenticate`, `get_reservations_kind`, `dump_reservation`) follow a *fail‑fast* approach: they log an error and invoke ``sys.exit(-1)`` on any abnormal condition. Consequently, ``main()`` does not need to catch exceptions from them. * The entry‑point guard ``if __name__ == "__main__":`` ensures the script runs only when executed directly, not when imported as a module. """ logger = logging.getLogger(__name__) coloredlogs.install() parser = argparse.ArgumentParser() parser.add_argument("-l", "--login", dest='login', type=str, required=False, help="Login") parser.add_argument("-C", "--city", dest='city', type=str, required=False, help="City.") parser.add_argument("-c", "--config", dest='config_filename', required=False, default=None, help="Configuration file.") parser.add_argument("-p", "--password", dest='password', nargs='?', required=False, default=None, help="Password.") parser.add_argument("-d", "--debug", dest='debug', action='store_true', required=False, help="Activate debug.") subparsers = parser.add_subparsers(title='subcommands', dest='command', help='subcommand help') subparsers.add_parser('A', help='Test authentication.') subparsers.add_parser('L', help='List all possible reservations types.') parserdump = subparsers.add_parser('D', help='Dump all reservation of some kind') parserdump.add_argument("-i", "--index", dest='index', type=int, required=True, help="Index of reservations to dump.") parserdump.add_argument("-o", "--output", dest='calendar', required=True, default='cal.ics', help="Output calendar file.") args = parser.parse_args() logger.debug("Initial arguments: %s", args) if args.debug: logger.info('Setting logging to debug mode') coloredlogs.set_level(level=logging.DEBUG) if args.config_filename is not None: try: with open(args.config_filename, 'r', encoding="utf-8") as config_file: config = configparser.ConfigParser() config.read_file(config_file) sections = config.sections() if 'Login' in sections: options = config.options('Login') if 'login' in options: args.login = config.get('Login','login') if 'password' in options: args.password = config.get('Login','password') if 'Espace' in sections: options = config.options('Espace') if 'ville' in options: args.city = config.get('Espace', 'ville') except IOError as e: logger.error('Impossible to open configuration file. Error: %s', e) sys.exit(-1) logger.debug("Final arguments: %s", args) if args.city is None: logger.error('City must be provided.') parser.print_help() sys.exit(-1) if args.login is None: logger.error('Login must be provided.') parser.print_help() sys.exit(-1) if args.password is None: args.password = getpass.getpass() base_url = 'https://www.espace-citoyens.net' # Authentication mainpage, cookies = authenticate(base_url, args.city, args.login, args.password) if (args.command is None) or (args.command == 'A'): sys.exit(0) # Switch between commands logger.info('Retrieve reservation kinds') resa_types = get_reservations_kind(base_url, mainpage, cookies) if args.command == 'L': resa_num = 1 for resa_type in resa_types: print(f"{resa_num:d} - {resa_type}") resa_num+=1 elif args.command == 'D': if args.index < 0: logger.error('Index can not be negative') sys.exit(-1) if args.index > len(resa_types): logger.error('Index is larger than number of reservations: %d', len(resa_types)) sys.exit(-1) resa_type = list(resa_types)[args.index-1] logger.info('Retrieve reservations for "%s"', resa_type) url = resa_types[resa_type] dump_reservation(base_url, args.city, url, cookies, args.calendar) if __name__ == "__main__": main()