diff --git a/.docker/README.md b/.docker/README.md index fb3c9e3..b710dfa 100644 --- a/.docker/README.md +++ b/.docker/README.md @@ -1,4 +1,6 @@ # Nyaa on Docker +> [!CAUTION] +> Docker deployment is out of date and currently unsupported in NyaaV3. Docker infrastructure is provided to ease setting up a dev environment diff --git a/.docker/es_sync_config.json b/.docker/es_sync_config.json index 2c4ec0c..a6c7f9e 100644 --- a/.docker/es_sync_config.json +++ b/.docker/es_sync_config.json @@ -4,7 +4,7 @@ "mysql_port": 3306, "mysql_user": "nyaadev", "mysql_password": "ZmtB2oihHFvc39JaEDoF", - "database": "nyaav2", + "database": "nyaav3", "internal_queue_depth": 10000, "es_chunk_size": 10000, "flush_interval": 5 diff --git a/.docker/full-stack.yml b/.docker/full-stack.yml index 4de5bb1..f2fb11b 100644 --- a/.docker/full-stack.yml +++ b/.docker/full-stack.yml @@ -48,7 +48,7 @@ services: - MYSQL_RANDOM_ROOT_PASSWORD=yes - MYSQL_USER=nyaadev - MYSQL_PASSWORD=ZmtB2oihHFvc39JaEDoF - - MYSQL_DATABASE=nyaav2 + - MYSQL_DATABASE=nyaav3 elasticsearch: image: elasticsearch:6.5.4 diff --git a/.docker/nyaa-config-partial.py b/.docker/nyaa-config-partial.py index f805c7c..65a7616 100644 --- a/.docker/nyaa-config-partial.py +++ b/.docker/nyaa-config-partial.py @@ -3,7 +3,7 @@ SITE_NAME = 'Nyaa [DEVEL]' GLOBAL_SITE_NAME = 'nyaa.devel' -SQLALCHEMY_DATABASE_URI = ('mysql://nyaadev:ZmtB2oihHFvc39JaEDoF@mariadb/nyaav2?charset=utf8mb4') +SQLALCHEMY_DATABASE_URI = ('mysql://nyaadev:ZmtB2oihHFvc39JaEDoF@mariadb/nyaav3?charset=utf8mb4') # MAIN_ANNOUNCE_URL = 'http://chihaya:6881/announce' # TRACKER_API_URL = 'http://chihaya:6881/api' BACKUP_TORRENT_FOLDER = '/nyaa-torrents' diff --git a/.github/issue_template.md b/.github/issue_template.md index dfe42b3..6419727 100644 --- a/.github/issue_template.md +++ b/.github/issue_template.md @@ -1,5 +1,3 @@ -Describe your issue/feature request here (you can remove all this text). Describe well and include images, if relevant! +Describe your issue/feature request here (you can remove all this text). Describe well and include images if relevant. -Please make sure to skim through the existing issues, your issue/request/etc may have already been noted! - -IMPORTANT: only submit issues that are relevant to the code. We do not offer support for any deployments of the project here; make your way to the IRC channel in such cases. +Please make sure to skim through the existing issues, as your issue/request/etc. may have already been noted! diff --git a/.travis.yml b/.travis.yml index 2615725..8c6f6c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,8 @@ language: python -python: "3.7" +python: "3.13" -dist: xenial -sudo: required +dist: jammy matrix: fast_finish: true @@ -14,7 +13,7 @@ services: mysql before_install: - - mysql -u root -e 'CREATE DATABASE nyaav2 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;' + - mysql -u root -e 'CREATE DATABASE nyaav3 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;' install: - pip install -r requirements.txt diff --git a/README.md b/README.md index c8c7bdc..e717905 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,54 @@ -# NyaaV2 [![Build Status](https://travis-ci.org/nyaadevs/nyaa.svg?branch=master)](https://travis-ci.org/nyaadevs/nyaa) +# NyaaV3 [![python](https://img.shields.io/badge/Python-3.13.2-3776AB.svg?style=flat&logo=python&logoColor=white)](https://www.python.org) ![Maintenance](https://img.shields.io/maintenance/yes/2025) ## Setting up for development -This project uses Python 3.7. There are features used that do not exist in 3.6, so make sure to use Python 3.7. -This guide also assumes you 1) are using Linux and 2) are somewhat capable with the commandline. -It's not impossible to run Nyaa on Windows, but this guide doesn't focus on that. +This project uses Python 3.13. The codebase has been updated from the original Python 3.7 version to use modern Python features and updated dependencies. +This guide assumes you are using Linux and are somewhat capable with the commandline. +Running Nyaa on Windows may be possible, but it's currently unsupported. + +### Major changes from NyaaV2 +- Updated from Python 3.7 to Python 3.13 +- Updated all dependencies to their latest versions +- Modernized code patterns for Flask 3.0 and SQLAlchemy 2.0 +- Replaced deprecated Flask-Script, orderedset and `flask.Markup` with Flask CLI, orderly-set and markupsafe +- Implemented mail error handling ### Code Quality: -- Before we get any deeper, remember to follow PEP8 style guidelines and run `./dev.py lint` before committing to see a list of warnings/problems. - - You may also use `./dev.py fix && ./dev.py isort` to automatically fix some of the issues reported by the previous command. +- Before we get any deeper, remember to follow PEP8 style guidelines and run `python dev.py lint` before committing to see a list of warnings/problems. + - You may also use `python dev.py fix && python dev.py isort` to automatically fix some of the issues reported by the previous command. - Other than PEP8, try to keep your code clean and easy to understand, as well. It's only polite! ### Running Tests The `tests` folder contains tests for the the `nyaa` module and the webserver. To run the tests: -- Make sure that you are in the python virtual environment. -- Run `./dev.py test` while in the repository directory. +- Make sure that you are in the Python virtual environment. +- Run `python dev.py test` while in the repository directory. ### Setting up Pyenv -pyenv eases the use of different Python versions, and as not all Linux distros offer 3.7 packages, it's right up our alley. -- Install dependencies https://github.com/pyenv/pyenv/wiki/Common-build-problems -- Install `pyenv` https://github.com/pyenv/pyenv/blob/master/README.md#installation -- Install `pyenv-virtualenv` https://github.com/pyenv/pyenv-virtualenv/blob/master/README.md -- Install Python 3.7.2 with `pyenv` and create a virtualenv for the project: - - `pyenv install 3.7.2` - - `pyenv virtualenv 3.7.2 nyaa` +pyenv eases the use of different Python versions, and as not all Linux distros offer 3.13 packages, it's right up our alley. +- Install [dependencies](https://github.com/pyenv/pyenv/wiki/Common-build-problems) +- Install [pyenv](https://github.com/pyenv/pyenv/blob/master/README.md#installation) +- Install [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv/blob/master/README.md) +- Install Python 3.13 with `pyenv` and create a virtualenv for the project: + - `pyenv install 3.13.2` + - `pyenv virtualenv 3.13.2 nyaa` - `pyenv activate nyaa` - Install dependencies with `pip install -r requirements.txt` - Copy `config.example.py` into `config.py` - Change `SITE_FLAVOR` in your `config.py` depending on which instance you want to host ### Setting up MySQL/MariaDB database -You *may* use SQLite but the current support for it in this project is outdated and rather unsupported. +> [!WARNING] +> You *may* use SQLite but it is currently untested and unsupported. - Enable `USE_MYSQL` flag in config.py -- Install latest mariadb by following instructions here https://downloads.mariadb.org/mariadb/repositories/ - - Tested versions: `mysql Ver 15.1 Distrib 10.0.30-MariaDB, for debian-linux-gnu (x86_64) using readline 5.2` +- Install MariaDB by following instructions [here](https://downloads.mariadb.org/mariadb/repositories/) - Run the following commands logged in as your root db user (substitute for your own `config.py` values if desired): - `CREATE USER 'test'@'localhost' IDENTIFIED BY 'test123';` - `GRANT ALL PRIVILEGES ON *.* TO 'test'@'localhost';` - `FLUSH PRIVILEGES;` - - `CREATE DATABASE nyaav2 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;` + - `CREATE DATABASE nyaav3 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;` ### Finishing up - Run `python db_create.py` to create the database and import categories - - Follow the advice of `db_create.py` and run `./db_migrate.py stamp head` to mark the database version for Alembic + - Follow the advice of `db_create.py` and run `python db_migrate.py stamp head` to mark the database version for Alembic - Start the dev server with `python run.py` - When you are finished developing, deactivate your virtualenv with `pyenv deactivate` or `source deactivate` (or just close your shell session) @@ -50,32 +57,35 @@ Continue below to learn about database migrations and enabling the advanced sear ## Database migrations -- Database migrations are done with [flask-Migrate](https://flask-migrate.readthedocs.io/), a wrapper around [Alembic](http://alembic.zzzcomputing.com/en/latest/). +> [!WARNING] +> The database migration feature has been updated but will no longer be supported in NyaaV3. +- Database migrations are done with [Flask-Migrate](https://flask-migrate.readthedocs.io/), a wrapper around [Alembic](http://alembic.zzzcomputing.com/en/latest/). +- The migration system has been updated to use Flask CLI instead of the deprecated Flask-Script. - If someone has made changes in the database schema and included a new migration script: - - If your database has never been marked by Alembic (you're on a database from before the migrations), run `./db_migrate.py stamp head` before pulling the new migration script(s). - - If you already have the new scripts, check the output of `./db_migrate.py history` instead and choose a hash that matches your current database state, then run `./db_migrate.py stamp `. + - If your database has never been marked by Alembic (you're on a database from before the migrations), run `python db_migrate.py db stamp head` before pulling the new migration script(s). + - If you already have the new scripts, check the output of `python db_migrate.py db history` instead and choose a hash that matches your current database state, then run `python db_migrate.py db stamp `. - Update your branch (eg. `git fetch && git rebase origin/master`) - - Run `./db_migrate.py upgrade head` to run the migration. Done! + - Run `python db_migrate.py db upgrade head` to run the migration. Done! - If *you* have made a change in the database schema: - Save your changes in `models.py` and ensure the database schema matches the previous version (ie. your new tables/columns are not added to the live database) - - Run `./db_migrate.py migrate -m "Short description of changes"` to automatically generate a migration script for the changes + - Run `python db_migrate.py db migrate -m "Short description of changes"` to automatically generate a migration script for the changes - Check the script (`migrations/versions/...`) and make sure it works! Alembic may not able to notice all changes. - - Run `./db_migrate.py upgrade` to run the migration and verify the upgrade works. - - (Run `./db_migrate.py downgrade` to verify the downgrade works as well, then upgrade again) + - Run `python db_migrate.py db upgrade` to run the migration and verify the upgrade works. + - (Run `python db_migrate.py db downgrade` to verify the downgrade works as well, then upgrade again) ## Setting up and enabling Elasticsearch ### Installing Elasticsearch - Install JDK with `sudo apt-get install openjdk-8-jdk` -- Install [Elasticsearch](https://www.elastic.co/downloads/elasticsearch) - - [From packages...](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html) +- Install Elasticsearch + - [From packages](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html) - Enable the service: - `sudo systemctl enable elasticsearch.service` - `sudo systemctl start elasticsearch.service` - - or [simply extracting the archives and running the files](https://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html), if you don't feel like permantently installing ES + - or [simply extracting the archives and running the files](https://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html), if you don't feel like permanently installing ES - Run `curl -XGET 'localhost:9200'` and make sure ES is running - - Optional: install [Kibana](https://www.elastic.co/products/kibana) as a search debug frontend for ES +- Install [Kibana](https://www.elastic.co/products/kibana) as a search debug frontend for ES (*optional*) ### Enabling MySQL Binlogging - Edit your MariaDB/MySQL server configuration and add the following under `[mariadb]`: @@ -113,3 +123,16 @@ However, take note that binglog is not necessary for simple ES testing and devel You're done! The script should now be feeding updates from the database to Elasticsearch. Take note, however, that the specified ES index refresh interval is 30 seconds, which may feel like a long time on local development. Feel free to adjust it or [poke Elasticsearch yourself!](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html) + +## License +This project is licensed under the GNU General Public License v3.0 (GPL-3.0). See the [LICENSE](LICENSE) file for more details. + +## Disclaimer +> [!CAUTION] +> **This project was created as a learning experience, and while it’s a torrent tracker, I can’t control how people choose to use it.** + +By using this software, you're agreeing to a few things: +- I'm not responsible for any legal issues that might come up from using this tracker, especially if it's used to share copyrighted content without permission. +- It's your responsibility to make sure you're following the laws in your area when using this software. + +**Please use this project wisely and stay on the right side of the law.** Happy coding! \ No newline at end of file diff --git a/WSGI.py b/WSGI.py index 6e080f8..83a2987 100644 --- a/WSGI.py +++ b/WSGI.py @@ -1,11 +1,16 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- +""" +WSGI entry point for the Nyaa application. +Compatible with Python 3.13. +""" import gevent.monkey gevent.monkey.patch_all() from nyaa import create_app +from flask import Flask -app = create_app('config') +app: Flask = create_app('config') if app.config['DEBUG']: from werkzeug.debug import DebuggedApplication diff --git a/config.example.py b/config.example.py index 253cb06..6904e4b 100644 --- a/config.example.py +++ b/config.example.py @@ -42,6 +42,12 @@ EXTERNAL_URLS = {'fap':'***', 'main':'***'} CSRF_SESSION_KEY = '***' SECRET_KEY = '***' +# Session cookie configuration +SESSION_COOKIE_NAME = 'nyaav3_session' +SESSION_COOKIE_SECURE = True +SESSION_COOKIE_HTTPONLY = True +SESSION_COOKIE_SAMESITE = 'Lax' + # Present a recaptcha for anonymous uploaders USE_RECAPTCHA = False # Require email validation @@ -82,7 +88,7 @@ RECAPTCHA_PRIVATE_KEY = '***' BASE_DIR = os.path.abspath(os.path.dirname(__file__)) if USE_MYSQL: - SQLALCHEMY_DATABASE_URI = ('mysql://test:test123@localhost/nyaav2?charset=utf8mb4') + SQLALCHEMY_DATABASE_URI = ('mysql://test:test123@localhost/nyaav3?charset=utf8mb4') else: SQLALCHEMY_DATABASE_URI = ( 'sqlite:///' + os.path.join(BASE_DIR, 'test.db') + '?check_same_thread=False') diff --git a/db_create.py b/db_create.py index 30fe4fe..0eaad9f 100755 --- a/db_create.py +++ b/db_create.py @@ -1,12 +1,19 @@ #!/usr/bin/env python3 +""" +Database creation script for Nyaa. +Compatible with Python 3.13 and SQLAlchemy 2.0. +""" +from typing import List, Tuple, Type + import sqlalchemy +from sqlalchemy import select from nyaa import create_app, models from nyaa.extensions import db app = create_app('config') -NYAA_CATEGORIES = [ +NYAA_CATEGORIES: List[Tuple[str, List[str]]] = [ ('Anime', ['Anime Music Video', 'English-translated', 'Non-English-translated', 'Raw']), ('Audio', ['Lossless', 'Lossy']), ('Literature', ['English-translated', 'Non-English-translated', 'Raw']), @@ -16,13 +23,23 @@ NYAA_CATEGORIES = [ ] -SUKEBEI_CATEGORIES = [ +SUKEBEI_CATEGORIES: List[Tuple[str, List[str]]] = [ ('Art', ['Anime', 'Doujinshi', 'Games', 'Manga', 'Pictures']), ('Real Life', ['Photobooks / Pictures', 'Videos']), ] -def add_categories(categories, main_class, sub_class): +def add_categories(categories: List[Tuple[str, List[str]]], + main_class: Type[models.MainCategoryBase], + sub_class: Type[models.SubCategoryBase]) -> None: + """ + Add categories to the database. + + Args: + categories: List of tuples containing main category name and list of subcategory names + main_class: Main category model class + sub_class: Subcategory model class + """ for main_cat_name, sub_cat_names in categories: main_cat = main_class(name=main_cat_name) for i, sub_cat_name in enumerate(sub_cat_names): @@ -36,19 +53,24 @@ if __name__ == '__main__': # Test for the user table, assume db is empty if it's not created database_empty = False try: - models.User.query.first() + stmt = select(models.User).limit(1) + db.session.execute(stmt).scalar_one_or_none() except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError): database_empty = True print('Creating all tables...') db.create_all() - nyaa_category_test = models.NyaaMainCategory.query.first() + # Check if Nyaa categories exist + stmt = select(models.NyaaMainCategory).limit(1) + nyaa_category_test = db.session.execute(stmt).scalar_one_or_none() if not nyaa_category_test: print('Adding Nyaa categories...') add_categories(NYAA_CATEGORIES, models.NyaaMainCategory, models.NyaaSubCategory) - sukebei_category_test = models.SukebeiMainCategory.query.first() + # Check if Sukebei categories exist + stmt = select(models.SukebeiMainCategory).limit(1) + sukebei_category_test = db.session.execute(stmt).scalar_one_or_none() if not sukebei_category_test: print('Adding Sukebei categories...') add_categories(SUKEBEI_CATEGORIES, models.SukebeiMainCategory, models.SukebeiSubCategory) diff --git a/db_migrate.py b/db_migrate.py index 92789f6..928ea1d 100755 --- a/db_migrate.py +++ b/db_migrate.py @@ -1,9 +1,14 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +""" +Database migration script for Nyaa. +Compatible with Python 3.13 and Flask-Migrate 4.0. +""" import sys +from typing import List -from flask_script import Manager -from flask_migrate import Migrate, MigrateCommand +from flask_migrate import Migrate +from flask.cli import FlaskGroup from nyaa import create_app from nyaa.extensions import db @@ -11,11 +16,17 @@ from nyaa.extensions import db app = create_app('config') migrate = Migrate(app, db) -manager = Manager(app) -manager.add_command("db", MigrateCommand) +def create_cli_app(): + return app + +cli = FlaskGroup(create_app=create_cli_app) if __name__ == "__main__": # Patch sys.argv to default to 'db' - sys.argv.insert(1, 'db') + if len(sys.argv) > 1 and sys.argv[1] not in ['--help', '-h']: + if sys.argv[1] not in ['db', 'routes', 'shell', 'run']: + args: List[str] = sys.argv.copy() + args.insert(1, 'db') + sys.argv = args - manager.run() + cli() diff --git a/dev.py b/dev.py index 7560784..d223370 100755 --- a/dev.py +++ b/dev.py @@ -4,8 +4,11 @@ This tool is designed to assist developers run common tasks, such as checking the code for lint issues, auto fixing some lint issues and running tests. It imports modules lazily (as-needed basis), so it runs faster! + +Compatible with Python 3.13. """ import sys +from typing import List, Optional, Generator, Any, Union LINT_PATHS = [ 'nyaa/', @@ -14,14 +17,14 @@ LINT_PATHS = [ TEST_PATHS = ['tests'] -def print_cmd(cmd, args): +def print_cmd(cmd: str, args: List[str]) -> None: """ Prints the command and args as you would run them manually. """ print('Running: {0}\n'.format( ' '.join([('\'' + a + '\'' if ' ' in a else a) for a in [cmd] + args]))) sys.stdout.flush() # Make sure stdout is flushed before continuing. -def check_config_values(): +def check_config_values() -> bool: """ Verify that all max_line_length values match. """ import configparser config = configparser.ConfigParser() @@ -32,7 +35,7 @@ def check_config_values(): autopep8 = config.get('pycodestyle', 'max_line_length', fallback=None) isort = config.get('isort', 'line_length', fallback=None) - values = (v for v in (flake8, autopep8, isort) if v is not None) + values: Generator[str, None, None] = (v for v in (flake8, autopep8, isort) if v is not None) found = next(values, False) if not found: print('Warning: No max line length setting set in setup.cfg.') @@ -44,7 +47,7 @@ def check_config_values(): return True -def print_help(): +def print_help() -> int: print('Nyaa Development Helper') print('=======================\n') print('Usage: {0} command [different arguments]'.format(sys.argv[0])) @@ -62,7 +65,7 @@ def print_help(): if __name__ == '__main__': - assert sys.version_info >= (3, 6), "Python 3.6 is required" + assert sys.version_info >= (3, 13), "Python 3.13 is required" check_config_values() diff --git a/es_sync_config.example.json b/es_sync_config.example.json index b2dc524..658c101 100644 --- a/es_sync_config.example.json +++ b/es_sync_config.example.json @@ -4,7 +4,7 @@ "mysql_port": 3306, "mysql_user": "nyaa", "mysql_password": "some_password", - "database": "nyaav2", + "database": "nyaav3", "internal_queue_depth": 10000, "es_chunk_size": 10000, "flush_interval": 5 diff --git a/import_to_es.py b/import_to_es.py index 6717100..fbd25b2 100755 --- a/import_to_es.py +++ b/import_to_es.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """ -Bulk load torents from mysql into elasticsearch `nyaav2` index, +Bulk load torents from mysql into elasticsearch `nyaav3` index, which is assumed to already exist. This is a one-shot deal, so you'd either need to complement it with a cron job or some binlog-reading thing (TODO) diff --git a/migrations/README b/migrations/README index 98e4f9c..e9c5557 100644 --- a/migrations/README +++ b/migrations/README @@ -1 +1,4 @@ +> [!WARNING] +> No longer supported in NyaaV3. + Generic single-database configuration. \ No newline at end of file diff --git a/nyaa/__init__.py b/nyaa/__init__.py index 241a6fb..e62ec22 100644 --- a/nyaa/__init__.py +++ b/nyaa/__init__.py @@ -1,8 +1,10 @@ import logging import os import string +from typing import Any, Optional import flask +from flask import Flask from flask_assets import Bundle # noqa F401 from nyaa.api_handler import api_blueprint @@ -18,11 +20,17 @@ from nyaa.views import register_views flask.url_for = caching_url_for -def create_app(config): +def create_app(config: Any) -> Flask: """ Nyaa app factory """ app = flask.Flask(__name__) app.config.from_object(config) + # Session cookie configuration + app.config['SESSION_COOKIE_NAME'] = 'nyaav3_session' + app.config['SESSION_COOKIE_SECURE'] = True + app.config['SESSION_COOKIE_HTTPONLY'] = True + app.config['SESSION_COOKIE_SAMESITE'] = 'Lax' + # Don't refresh cookie each request app.config['SESSION_REFRESH_EACH_REQUEST'] = False @@ -34,24 +42,24 @@ def create_app(config): # Forbid caching @app.after_request - def forbid_cache(request): - request.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate, max-age=0' - request.headers['Pragma'] = 'no-cache' - request.headers['Expires'] = '0' - return request + def forbid_cache(response: flask.Response) -> flask.Response: + response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate, max-age=0' + response.headers['Pragma'] = 'no-cache' + response.headers['Expires'] = '0' + return response # Add a timer header to the requests when debugging # This gives us a simple way to benchmark requests off-app import time @app.before_request - def timer_before_request(): + def timer_before_request() -> None: flask.g.request_start_time = time.time() @app.after_request - def timer_after_request(request): - request.headers['X-Timer'] = time.time() - flask.g.request_start_time - return request + def timer_after_request(response: flask.Response) -> flask.Response: + response.headers['X-Timer'] = str(time.time() - flask.g.request_start_time) + return response else: app.logger.setLevel(logging.WARNING) @@ -63,17 +71,17 @@ def create_app(config): app.config['LOG_FILE'], maxBytes=10000, backupCount=1) app.logger.addHandler(app.log_handler) - # Log errors and display a message to the user in production mdode + # Log errors and display a message to the user in production mode if not app.config['DEBUG']: @app.errorhandler(500) - def internal_error(exception): + def internal_error(exception: Exception) -> flask.Response: random_id = random_string(8, string.ascii_uppercase + string.digits) # Pst. Not actually unique, but don't tell anyone! - app.logger.error('Exception occurred! Unique ID: %s', random_id, exc_info=exception) + app.logger.error(f'Exception occurred! Unique ID: {random_id}', exc_info=exception) markup_source = ' '.join([ 'An error occurred!', 'Debug information has been logged.', - 'Please pass along this ID: {}'.format(random_id) + f'Please pass along this ID: {random_id}' ]) flask.flash(flask.Markup(markup_source), 'danger') @@ -101,10 +109,15 @@ def create_app(config): app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False app.config['MYSQL_DATABASE_CHARSET'] = 'utf8mb4' db.init_app(app) + + # Import the fixed Ban.banned method + with app.app_context(): + import nyaa.fixed_ban # Assets assets.init_app(app) - assets._named_bundles = {} # Hack to fix state carrying over in tests + if hasattr(assets, '_named_bundles'): + assets._named_bundles = {} # Hack to fix state carrying over in tests main_js = Bundle('js/main.js', filters='rjsmin', output='js/main.min.js') bs_js = Bundle('js/bootstrap-select.js', filters='rjsmin', output='js/bootstrap-select.min.js') diff --git a/nyaa/backend.py b/nyaa/backend.py index e0b0aa7..c8bb0ac 100644 --- a/nyaa/backend.py +++ b/nyaa/backend.py @@ -5,10 +5,10 @@ from datetime import datetime, timedelta from ipaddress import ip_address import flask -from werkzeug import secure_filename +from werkzeug.utils import secure_filename import sqlalchemy -from orderedset import OrderedSet +from orderly_set import OrderlySet from nyaa import models, utils from nyaa.extensions import db @@ -304,7 +304,7 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False): db.session.flush() # Store the users trackers - trackers = OrderedSet() + trackers = OrderlySet() announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii') if announce: trackers.add(announce) @@ -319,12 +319,12 @@ def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False): webseed_list = torrent_data.torrent_dict.get('url-list') or [] if isinstance(webseed_list, bytes): webseed_list = [webseed_list] # qB doesn't contain a sole url in a list - webseeds = OrderedSet(webseed.decode('utf-8') for webseed in webseed_list) + webseeds = OrderlySet(webseed.decode('utf-8') for webseed in webseed_list) # Remove our trackers, maybe? TODO ? # Search for/Add trackers in DB - db_trackers = OrderedSet() + db_trackers = OrderlySet() for announce in trackers: tracker = models.Trackers.by_uri(announce) diff --git a/nyaa/custom_pagination.py b/nyaa/custom_pagination.py new file mode 100644 index 0000000..f83be5d --- /dev/null +++ b/nyaa/custom_pagination.py @@ -0,0 +1,99 @@ +from typing import Any, List, Optional, Sequence, TypeVar, Union + +T = TypeVar('T') + +class CustomPagination: + """ + A custom pagination class that mimics the interface of Flask-SQLAlchemy's Pagination + but doesn't rely on the _query_items method. + """ + + def __init__(self, query: Any, page: int, per_page: int, total: int, items: List[T]): + """ + Initialize a new CustomPagination object. + + Args: + query: The query object (not used, but kept for compatibility) + page: The current page number (1-indexed) + per_page: The number of items per page + total: The total number of items + items: The items on the current page + """ + self.query = query + self.page = page + self.per_page = per_page + self.total = total + self.items = items + + # For compatibility with LimitedPagination + self.actual_count = total + + @property + def has_prev(self) -> bool: + """Return True if there is a previous page.""" + return self.page > 1 + + @property + def has_next(self) -> bool: + """Return True if there is a next page.""" + return self.page < self.pages + + @property + def pages(self) -> int: + """The total number of pages.""" + if self.per_page == 0 or self.total == 0: + return 0 + return max(1, (self.total + self.per_page - 1) // self.per_page) + + @property + def prev_num(self) -> Optional[int]: + """The previous page number, or None if this is the first page.""" + if self.has_prev: + return self.page - 1 + return None + + @property + def next_num(self) -> Optional[int]: + """The next page number, or None if this is the last page.""" + if self.has_next: + return self.page + 1 + return None + + @property + def first(self) -> int: + """The number of the first item on the page, starting from 1, or 0 if there are no items.""" + if not self.items: + return 0 + return (self.page - 1) * self.per_page + 1 + + @property + def last(self) -> int: + """The number of the last item on the page, starting from 1, inclusive, or 0 if there are no items.""" + if not self.items: + return 0 + return min(self.total, self.page * self.per_page) + + def iter_pages(self, left_edge: int = 2, left_current: int = 2, + right_current: int = 5, right_edge: int = 2) -> Sequence[Optional[int]]: + """ + Yield page numbers for a pagination widget. + + Skipped pages between the edges and middle are represented by a None. + """ + last = 0 + for num in range(1, self.pages + 1): + if (num <= left_edge or + (num > self.page - left_current - 1 and num < self.page + right_current) or + num > self.pages - right_edge): + if last + 1 != num: + yield None + yield num + last = num + + def __iter__(self): + """Iterate over the items on the current page.""" + return iter(self.items) + + def __len__(self): + """Return the number of items on the current page.""" + return len(self.items) diff --git a/nyaa/email.py b/nyaa/email.py index a74a631..f0a997c 100644 --- a/nyaa/email.py +++ b/nyaa/email.py @@ -45,29 +45,58 @@ class EmailHolder(object): def send_email(email_holder): + """Send an email using the configured mail backend.""" mail_backend = app.config.get('MAIL_BACKEND') - if mail_backend == 'mailgun': - _send_mailgun(email_holder) - elif mail_backend == 'smtp': - _send_smtp(email_holder) - elif mail_backend: - # TODO: Do this in logging.error when we have that set up - print('Unknown mail backend:', mail_backend) + + if not mail_backend: + app.logger.warning('No mail backend configured, skipping email send') + return False + + try: + if mail_backend == 'mailgun': + success = _send_mailgun(email_holder) + elif mail_backend == 'smtp': + success = _send_smtp(email_holder) + else: + app.logger.error(f'Unknown mail backend: {mail_backend}') + return False + + if not success: + app.logger.error(f'Failed to send email using {mail_backend} backend') + return False + + app.logger.info(f'Email successfully sent using {mail_backend} backend') + return True + + except Exception as e: + app.logger.error(f'Error sending email: {str(e)}') + return False def _send_mailgun(email_holder): - mailgun_endpoint = app.config['MAILGUN_API_BASE'] + '/messages' - auth = ('api', app.config['MAILGUN_API_KEY']) - data = { - 'from': app.config['MAIL_FROM_ADDRESS'], - 'to': email_holder.format_recipient(), - 'subject': email_holder.subject, - 'text': email_holder.text, - 'html': email_holder.html - } - r = requests.post(mailgun_endpoint, data=data, auth=auth) - # TODO real error handling? - assert r.status_code == 200 + """Send an email using Mailgun API with proper error handling.""" + try: + mailgun_endpoint = app.config['MAILGUN_API_BASE'] + '/messages' + auth = ('api', app.config['MAILGUN_API_KEY']) + data = { + 'from': app.config['MAIL_FROM_ADDRESS'], + 'to': email_holder.format_recipient(), + 'subject': email_holder.subject, + 'text': email_holder.text, + 'html': email_holder.html + } + + r = requests.post(mailgun_endpoint, data=data, auth=auth) + + if r.status_code != 200: + app.logger.error(f'Mailgun API error: {r.status_code} - {r.text}') + return False + + return True + + except Exception as e: + app.logger.error(f'Error sending email via Mailgun: {str(e)}') + return False def _send_smtp(email_holder): diff --git a/nyaa/extensions.py b/nyaa/extensions.py index 1c6498f..44753d6 100644 --- a/nyaa/extensions.py +++ b/nyaa/extensions.py @@ -1,4 +1,5 @@ import os.path +from typing import Any, Optional, Sequence, TypeVar, Union from flask import abort from flask.config import Config @@ -7,7 +8,9 @@ from flask_caching import Cache from flask_debugtoolbar import DebugToolbarExtension from flask_limiter import Limiter from flask_limiter.util import get_remote_address -from flask_sqlalchemy import BaseQuery, Pagination, SQLAlchemy +from flask_sqlalchemy import SQLAlchemy +from flask_sqlalchemy.pagination import Pagination +from sqlalchemy.orm import Query assets = Environment() db = SQLAlchemy() @@ -15,16 +18,28 @@ toolbar = DebugToolbarExtension() cache = Cache() limiter = Limiter(key_func=get_remote_address) +# Type variable for query results +T = TypeVar('T') + class LimitedPagination(Pagination): - def __init__(self, actual_count, *args, **kwargs): + def __init__(self, actual_count: int, *args: Any, **kwargs: Any) -> None: self.actual_count = actual_count super().__init__(*args, **kwargs) -def fix_paginate(): - - def paginate_faste(self, page=1, per_page=50, max_page=None, step=5, count_query=None): +def fix_paginate() -> None: + """Add custom pagination method to SQLAlchemy Query.""" + + def paginate_faste( + self: Query[T], + page: int = 1, + per_page: int = 50, + max_page: Optional[int] = None, + step: int = 5, + count_query: Optional[Query[int]] = None + ) -> LimitedPagination: + """Custom pagination that supports max_page and count_query.""" if page < 1: abort(404) @@ -36,6 +51,10 @@ def fix_paginate(): total_query_count = count_query.scalar() else: total_query_count = self.count() + + if total_query_count is None: + total_query_count = 0 + actual_query_count = total_query_count if max_page: total_query_count = min(total_query_count, max_page * per_page) @@ -49,17 +68,20 @@ def fix_paginate(): return LimitedPagination(actual_query_count, self, page, per_page, total_query_count, items) - BaseQuery.paginate_faste = paginate_faste + # Monkey patch the Query class + setattr(Query, 'paginate_faste', paginate_faste) -def _get_config(): - # Workaround to get an available config object before the app is initiallized - # Only needed/used in top-level and class statements - # https://stackoverflow.com/a/18138250/7597273 +def _get_config() -> Config: + """ + Workaround to get an available config object before the app is initialized. + Only needed/used in top-level and class statements. + https://stackoverflow.com/a/18138250/7597273 + """ root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) - config = Config(root_path) - config.from_object('config') - return config + config_obj = Config(root_path) + config_obj.from_object('config') + return config_obj config = _get_config() diff --git a/nyaa/fixed_ban.py b/nyaa/fixed_ban.py new file mode 100644 index 0000000..016ea67 --- /dev/null +++ b/nyaa/fixed_ban.py @@ -0,0 +1,26 @@ +from typing import Optional, Union +from sqlalchemy import or_, select +from nyaa.extensions import db +from nyaa.models import Ban + +# Fix the banned method to return a query object instead of a list +@classmethod +def fixed_banned(cls, user_id: Optional[int], user_ip: Optional[bytes]): + """Check if a user or IP is banned. + + Returns a query object that can be further filtered or used with .first(), .all(), etc. + """ + if not user_id and not user_ip: + # Return an empty query that will return no results + return db.session.query(cls).filter(cls.id < 0) + + clauses = [] + if user_id: + clauses.append(cls.user_id == user_id) + if user_ip: + clauses.append(cls.user_ip == user_ip) + + return db.session.query(cls).filter(or_(*clauses)) + +# Replace the original method with our fixed version +Ban.banned = fixed_banned diff --git a/nyaa/forms.py b/nyaa/forms.py index af15003..ddf1526 100644 --- a/nyaa/forms.py +++ b/nyaa/forms.py @@ -11,7 +11,8 @@ from wtforms import (BooleanField, HiddenField, PasswordField, SelectField, Stri SubmitField, TextAreaField) from wtforms.validators import (DataRequired, Email, EqualTo, Length, Optional, Regexp, StopValidation, ValidationError) -from wtforms.widgets import HTMLString # For DisabledSelectField +# from wtforms.widgets import HTMLString # For DisabledSelectField +from markupsafe import Markup from wtforms.widgets import Select as SelectWidget # For DisabledSelectField from wtforms.widgets import html_params @@ -223,7 +224,7 @@ class DisabledSelectWidget(SelectWidget): extra = disabled and {'disabled': ''} or {} html.append(self.render_option(val, label, selected, **extra)) html.append('') - return HTMLString(''.join(html)) + return Markup(''.join(html)) class DisabledSelectField(SelectField): @@ -265,7 +266,7 @@ class InlineButtonWidget(object): kwargs.setdefault('type', self.input_type) if not label: label = field.label.text - return HTMLString('