From 0c7733ca839446b8e54ac639422eff2f5540f200 Mon Sep 17 00:00:00 2001 From: sb745 <201226723+sb745@users.noreply.github.com> Date: Mon, 22 Dec 2025 02:13:36 +0200 Subject: [PATCH 1/6] Update dependencies and config example --- README.md | 14 ++++----- config.example.py | 4 +-- requirements.txt | 74 +++++++++++++++++++++++------------------------ 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 854bc11..24e497a 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -# NyaaV3 [![python](https://img.shields.io/badge/Python-3.13-3776AB.svg?style=flat&logo=python&logoColor=white)](https://www.python.org) ![Maintenance](https://img.shields.io/maintenance/yes/2025) +# NyaaV3 [![python](https://img.shields.io/badge/Python-3.14-3776AB.svg?style=flat&logo=python&logoColor=white)](https://www.python.org) ![Maintenance](https://img.shields.io/maintenance/yes/2025) ## Setting up for development -This project uses Python 3.13. The codebase has been updated from the original Python 3.7 version to use modern Python features and updated dependencies. +This project uses Python 3.14. The codebase has been updated from the original Python 3.7 version to use modern Python features and updated dependencies. This guide assumes you are using Linux and are somewhat capable with the commandline. Running Nyaa on Windows may be possible, but it's currently unsupported. ### Major changes from NyaaV2 -- Updated from Python 3.7 to Python 3.13 +- Updated from Python 3.7 to Python 3.14 - Updated all dependencies to their latest versions - Modernized code patterns for Flask 3.0 and SQLAlchemy 2.0 - Replaced deprecated Flask-Script, orderedset and `flask.Markup` with Flask CLI, orderly-set and markupsafe @@ -23,13 +23,13 @@ The `tests` folder contains tests for the the `nyaa` module and the webserver. T - Run `python dev.py test` while in the repository directory. ### Setting up Pyenv -pyenv eases the use of different Python versions, and as not all Linux distros offer 3.13 packages, it's right up our alley. +pyenv eases the use of different Python versions, and as not all Linux distros offer 3.14 packages, it's right up our alley. - Install [dependencies](https://github.com/pyenv/pyenv/wiki/Common-build-problems) - Install [pyenv](https://github.com/pyenv/pyenv/blob/master/README.md#installation) - Install [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv/blob/master/README.md) -- Install Python 3.13 with `pyenv` and create a virtualenv for the project: - - `pyenv install 3.13.2` - - `pyenv virtualenv 3.13.2 nyaa` +- Install Python 3.14 with `pyenv` and create a virtualenv for the project: + - `pyenv install 3.14.2` + - `pyenv virtualenv 3.14.2 nyaa` - `pyenv activate nyaa` - Install dependencies with `pip install -r requirements.txt` - Copy `config.example.py` into `config.py` diff --git a/config.example.py b/config.example.py index 014e0e8..6c24285 100644 --- a/config.example.py +++ b/config.example.py @@ -88,7 +88,7 @@ RECAPTCHA_PRIVATE_KEY = '***' BASE_DIR = os.path.abspath(os.path.dirname(__file__)) if USE_MYSQL: - SQLALCHEMY_DATABASE_URI = ('mysql://test:test123@localhost/nyaav3?charset=utf8mb4') + SQLALCHEMY_DATABASE_URI = ('mysql://nyaauser:nyaapass@localhost/nyaav3?charset=utf8mb4') else: SQLALCHEMY_DATABASE_URI = ( 'sqlite:///' + os.path.join(BASE_DIR, 'test.db') + '?check_same_thread=False') @@ -98,7 +98,7 @@ else: ########### # 'smtp' or 'mailgun' -MAIL_BACKEND = 'mailgun' +MAIL_BACKEND = 'smtp' MAIL_FROM_ADDRESS = 'Sender Name ' # Mailgun settings diff --git a/requirements.txt b/requirements.txt index 68cb452..102efad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ -alembic==1.14.1 +alembic==1.17.2 appdirs==1.4.4 -argon2-cffi==23.1.0 +argon2-cffi==25.1.0 autopep8==2.3.2 blinker==1.9.0 -cffi==1.17.1 -click==8.1.8 -dnspython==2.7.0 -elasticsearch==8.17.1 -elasticsearch-dsl==8.17.1 -flake8==7.1.2 -flake8-isort==6.1.2 -Flask==3.1.0 +cffi==2.0.0 +click==8.3.1 +dnspython==2.8.0 +elasticsearch==8.19.2 +elasticsearch-dsl==8.18.0 +flake8==7.3.0 +flake8-isort==7.0.0 +Flask==3.1.2 Flask-Assets==2.1.0 Flask-DebugToolbar==0.16.0 Flask-Migrate==4.1.0 @@ -18,42 +18,42 @@ flask-paginate==2024.4.12 # Flask-Script removed as it's deprecated and replaced with Flask CLI Flask-SQLAlchemy==3.1.1 Flask-WTF==1.2.2 -gevent==24.11.1 -greenlet==3.1.1 -isort==6.0.1 +gevent==25.9.1 +greenlet==3.3.0 +isort==7.0.0 itsdangerous==2.2.0 -Jinja2==3.1.5 -Mako==1.3.9 -MarkupSafe==3.0.2 -mysql-replication==1.0.9 +Jinja2==3.1.6 +Mako==1.3.10 +MarkupSafe==3.0.3 +mysql-replication==1.0.12 mysqlclient==2.2.7 # orderedset removed as it's deprecated and replaced with Flask CLI -orderly-set==5.3.0 -packaging==24.2 +orderly-set==5.5.0 +packaging==25.0 passlib==1.7.4 progressbar33==2.4 py==1.11.0 -pycodestyle==2.12.1 -pycparser==2.22 -PyMySQL==1.1.1 -pyparsing==3.2.1 -pytest==8.3.4 -python-dateutil==2.9.0 +pycodestyle==2.14.0 +pycparser==2.23 +PyMySQL==1.1.2 +pyparsing==3.2.5 +pytest==9.0.2 +python-dateutil==2.9.0.post0 python-editor==1.0.4 python-utils==3.9.1 -requests==2.32.3 -SQLAlchemy==2.0.38 +requests==2.32.5 +SQLAlchemy==2.0.45 SQLAlchemy-FullText-Search==0.3.0 -SQLAlchemy-Utils==0.41.2 +SQLAlchemy-Utils==0.42.1 statsd==4.0.1 -urllib3==2.3.0 -uWSGI==2.0.28 -redis==5.2.1 -webassets==2.0 -Werkzeug==3.1.3 +urllib3==2.6.2 +uWSGI==2.0.31 +redis==7.1.0 +webassets==3.0.0 +Werkzeug==3.1.4 WTForms==3.2.1 Flask-Caching==2.3.1 -Flask-Limiter==3.10.1 -mypy==1.15.0 -typing-extensions==4.12.2 -email-validator==2.2.0 \ No newline at end of file +Flask-Limiter==4.1.1 +mypy==1.19.1 +typing-extensions==4.15.0 +email-validator==2.3.0 \ No newline at end of file From 4fe13c1df3551b01353d191041cb856d8a2ec548 Mon Sep 17 00:00:00 2001 From: sb745 <201226723+sb745@users.noreply.github.com> Date: Mon, 22 Dec 2025 03:35:11 +0200 Subject: [PATCH 2/6] Fixed Elasticsearch --- README.md | 14 +++++++++----- config.example.py | 2 +- es_sync_config.example.json | 4 ++-- import_to_es.py | 8 +++++--- sync_es.py | 8 ++++---- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 24e497a..f1a8af0 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Running Nyaa on Windows may be possible, but it's currently unsupported. ### Major changes from NyaaV2 - Updated from Python 3.7 to Python 3.14 -- Updated all dependencies to their latest versions +- Updated all dependencies - Modernized code patterns for Flask 3.0 and SQLAlchemy 2.0 - Replaced deprecated Flask-Script, orderedset and `flask.Markup` with Flask CLI, orderly-set and markupsafe - Implemented mail error handling @@ -77,7 +77,7 @@ Continue below to learn about database migrations and enabling the advanced sear ## Setting up and enabling Elasticsearch ### Installing Elasticsearch -- Install JDK with `sudo apt-get install openjdk-8-jdk` +- Install JDK with `sudo apt-get install openjdk-21-jdk` - Install Elasticsearch - [From packages](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html) - Enable the service: @@ -85,6 +85,7 @@ Continue below to learn about database migrations and enabling the advanced sear - `sudo systemctl start elasticsearch.service` - or [simply extracting the archives and running the files](https://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html), if you don't feel like permanently installing ES - Run `curl -XGET 'localhost:9200'` and make sure ES is running + - You may need to set `xpack.security.enabled: false` in your `elasticsearch.yml` file if curl output is empty - Install [Kibana](https://www.elastic.co/products/kibana) as a search debug frontend for ES (*optional*) ### Enabling MySQL Binlogging @@ -99,7 +100,10 @@ Continue below to learn about database migrations and enabling the advanced sear - Copy the example configuration (`es_sync_config.example.json`) as `es_sync_config.json` and adjust options in it to your liking (verify the connection options!) - Connect to mysql as root - Verify that the result of `SHOW VARIABLES LIKE 'binlog_format';` is `ROW` - - Execute `GRANT REPLICATION SLAVE ON *.* TO 'username'@'localhost';` to allow your configured user access to the binlog + - Execute `GRANT REPLICATION SLAVE ON *.* TO 'nyaav3'@'localhost';` to allow your configured user access to the binlog and one of the following: + - For MySQL: `GRANT REPLICATION CLIENT ON *.* TO 'nyaauser'@'localhost';` + - For MariaDB: `GRANT BINLOG MONITOR ON *.* TO 'nyaauser'@'localhost';` +- Run `./create_es.sh` to create the indices for the torrents: `nyaa` and `sukebei` ### Setting up ES - Run `./create_es.sh` to create the indices for the torrents: `nyaa` and `sukebei` @@ -116,8 +120,8 @@ However, take note that binglog is not necessary for simple ES testing and devel ### Setting up sync_es.py `sync_es.py` keeps the Elasticsearch indices updated by reading the binlog and pushing the changes to the ES indices. -- Make sure `es_sync_config.json` is configured with the user you grated the `REPLICATION` permissions -- Run `import_to_es.py` and copy the outputted JSON into the file specified by `save_loc` in your `es_sync_config.json` +- Make sure `es_sync_config.json` is configured with the user you granted the `REPLICATION` permissions +- Run `python import_to_es.py /path/to/file.json` and copy the outputted JSON into the file specified by `save_loc` in your `es_sync_config.json` file - Run `sync_es.py` as-is *or*, for actual deployment, set it up as a service and run it, preferably as the system/root - Make sure `sync_es.py` runs within the venv with the right dependencies! diff --git a/config.example.py b/config.example.py index 6c24285..48bde20 100644 --- a/config.example.py +++ b/config.example.py @@ -180,7 +180,7 @@ ES_MAX_SEARCH_RESULT = 1000 # ES index name generally (nyaa or sukebei) ES_INDEX_NAME = SITE_FLAVOR # ES hosts -ES_HOSTS = ['localhost:9200'] +ES_HOSTS = ['http://localhost:9200'] ################ ## Commenting ## diff --git a/es_sync_config.example.json b/es_sync_config.example.json index 658c101..2ccbd6e 100644 --- a/es_sync_config.example.json +++ b/es_sync_config.example.json @@ -2,8 +2,8 @@ "save_loc": "/tmp/pos.json", "mysql_host": "127.0.0.1", "mysql_port": 3306, - "mysql_user": "nyaa", - "mysql_password": "some_password", + "mysql_user": "nyaauser", + "mysql_password": "nyaapass", "database": "nyaav3", "internal_queue_depth": 10000, "es_chunk_size": 10000, diff --git a/import_to_es.py b/import_to_es.py index fbd25b2..6ce5326 100755 --- a/import_to_es.py +++ b/import_to_es.py @@ -13,12 +13,13 @@ import progressbar from elasticsearch import Elasticsearch from elasticsearch.client import IndicesClient from elasticsearch import helpers +from sqlalchemy import text from nyaa import create_app, models from nyaa.extensions import db app = create_app('config') -es = Elasticsearch(hosts=app.config['ES_HOSTS'], timeout=30) +es = Elasticsearch(hosts=app.config['ES_HOSTS'], request_timeout=30) ic = IndicesClient(es) def pad_bytes(in_bytes, size): @@ -98,14 +99,15 @@ FLAVORS = [ # Get binlog status from mysql with app.app_context(): - master_status = db.engine.execute('SHOW MASTER STATUS;').fetchone() + with db.engine.begin() as connection: + master_status = connection.execute(text('SHOW MASTER STATUS;')).fetchone() position_json = { 'log_file': master_status[0], 'log_pos': master_status[1] } - print('Save the following in the file configured in your ES sync config JSON:') + print('Save the following in the file configured in es_sync_config.json:') print(json.dumps(position_json)) for flavor, torrent_class in FLAVORS: diff --git a/sync_es.py b/sync_es.py index aa1adcb..090c792 100755 --- a/sync_es.py +++ b/sync_es.py @@ -68,9 +68,9 @@ stats = StatsClient('localhost', 8125, prefix="sync_es") SAVE_LOC = config.get('save_loc', "/tmp/pos.json") MYSQL_HOST = config.get('mysql_host', '127.0.0.1') MYSQL_PORT = config.get('mysql_port', 3306) -MYSQL_USER = config.get('mysql_user', 'root') -MYSQL_PW = config.get('mysql_password', 'dunnolol') -NT_DB = config.get('database', 'nyaav2') +MYSQL_USER = config.get('mysql_user', 'nyaauser') +MYSQL_PW = config.get('mysql_password', 'nyaapass') +NT_DB = config.get('database', 'nyaav3') INTERNAL_QUEUE_DEPTH = config.get('internal_queue_depth', 10000) ES_CHUNK_SIZE = config.get('es_chunk_size', 10000) # seconds since no events happening to flush to es. remember this also @@ -263,7 +263,7 @@ class EsPoster(ExitingThread): self.flush_interval = flush_interval def run_happy(self): - es = Elasticsearch(hosts=app.config['ES_HOSTS'], timeout=30) + es = Elasticsearch(hosts=app.config['ES_HOSTS'], request_timeout=30) last_save = time.time() since_last = 0 From 653cf1af862e7be84646989265f2de6a80a9ec64 Mon Sep 17 00:00:00 2001 From: sb745 <201226723+sb745@users.noreply.github.com> Date: Mon, 22 Dec 2025 03:48:50 +0200 Subject: [PATCH 3/6] Minor wording change --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f1a8af0..4f99a2c 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Continue below to learn about database migrations and enabling the advanced sear ## Database migrations > [!WARNING] > The database migration feature has been updated but will no longer be supported in NyaaV3. -- Database migrations are done with [Flask-Migrate](https://flask-migrate.readthedocs.io/), a wrapper around [Alembic](http://alembic.zzzcomputing.com/en/latest/). +- Database migrations are done using [Flask-Migrate](https://flask-migrate.readthedocs.io/), a wrapper around [Alembic](http://alembic.zzzcomputing.com/en/latest/). - The migration system has been updated to use Flask CLI instead of the deprecated Flask-Script. - If someone has made changes in the database schema and included a new migration script: - If your database has never been marked by Alembic (you're on a database from before the migrations), run `python db_migrate.py db stamp head` before pulling the new migration script(s). From 27cc6568429020faa2671e47c0cd4c1d7f8b0acc Mon Sep 17 00:00:00 2001 From: sb745 <201226723+sb745@users.noreply.github.com> Date: Mon, 22 Dec 2025 04:08:31 +0200 Subject: [PATCH 4/6] Incomplete multi-character sanitization fix Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- nyaa/static/js/bootstrap-select.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nyaa/static/js/bootstrap-select.js b/nyaa/static/js/bootstrap-select.js index ee12bb6..c5c9aa6 100644 --- a/nyaa/static/js/bootstrap-select.js +++ b/nyaa/static/js/bootstrap-select.js @@ -804,8 +804,10 @@ title = typeof this.options.title !== 'undefined' ? this.options.title : this.options.noneSelectedText; } - //strip all HTML tags and trim the result, then unescape any escaped tags - this.$button.attr('title', htmlUnescape($.trim(title.replace(/<[^>]*>?/g, '')))); + //strip all HTML tags in a DOM-safe way and trim the result, then unescape any escaped tags + var $tmp = $('
').html(title); + var plainTitle = $tmp.text(); + this.$button.attr('title', htmlUnescape($.trim(plainTitle))); this.$button.children('.filter-option').html(title); this.$element.trigger('rendered.bs.select'); From ff4dc572f3b4f4c33463705bc0e4c02f686c8f93 Mon Sep 17 00:00:00 2001 From: sb745 <201226723+sb745@users.noreply.github.com> Date: Mon, 22 Dec 2025 05:54:03 +0200 Subject: [PATCH 5/6] Added security policy --- SECURITY.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..1691d31 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,12 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 2025-12 | :white_check_mark: | +| 2025-03 | :x: | + +## Reporting a Vulnerability + +Please report high risk security vulnerabilities privately. \ No newline at end of file From e50efa5449bb6bc1f9867b92f973bddd26f18deb Mon Sep 17 00:00:00 2001 From: sb745 <201226723+sb745@users.noreply.github.com> Date: Mon, 22 Dec 2025 13:58:52 +0200 Subject: [PATCH 6/6] Added commit hash explanation --- config.example.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.example.py b/config.example.py index 48bde20..425e27c 100644 --- a/config.example.py +++ b/config.example.py @@ -232,5 +232,6 @@ CACHE_THRESHOLD = 8192 # RATELIMIT_STORAGE_URL="redis://host:port" RATELIMIT_KEY_PREFIX="nyaaratelimit_" -# Use this to show the commit hash in the footer (see layout.html) +# The commit hash is automatically shown in the footer if .git folder is present (see layout.html) +# Use this to manually show the hash # COMMIT_HASH="[enter your commit hash here]"; \ No newline at end of file