From 0af498f98faa1b5107b1d9d98cf0ca58e469564a Mon Sep 17 00:00:00 2001 From: stringertheory Date: Mon, 29 Jan 2024 17:53:03 -0600 Subject: [PATCH] omg what a pain --- Makefile | 2 +- README.md | 50 + clean_links/clean.py | 101 ++ clean_links/clearurls.json | 2559 +++++++++++++++++++++++++++++ clean_links/clearurls_config.json | 17 + clean_links/config.py | 34 + clean_links/unshorten.py | 85 + poetry.lock | 16 +- pyproject.toml | 6 +- 9 files changed, 2867 insertions(+), 3 deletions(-) create mode 100644 clean_links/clean.py create mode 100644 clean_links/clearurls.json create mode 100644 clean_links/clearurls_config.json create mode 100644 clean_links/config.py create mode 100644 clean_links/unshorten.py diff --git a/Makefile b/Makefile index 2a01119..7487ee2 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ check: ## Run code quality tools. @echo "🚀 Linting code: Running pre-commit" @poetry run pre-commit run -a @echo "🚀 Static type checking: Running mypy" - @poetry run mypy + @poetry run mypy --disable-error-code attr-defined .PHONY: test test: ## Test the code with pytest diff --git a/README.md b/README.md index e69de29..a4ebd46 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,50 @@ +# clean-links + +[![Release](https://img.shields.io/github/v/release/stringertheory/clean-links)](https://img.shields.io/github/v/release/stringertheory/clean-links) +[![Build status](https://img.shields.io/github/actions/workflow/status/stringertheory/clean-links/main.yml?branch=main)](https://github.com/stringertheory/clean-links/actions/workflows/main.yml?query=branch%3Amain) +[![codecov](https://codecov.io/gh/stringertheory/clean-links/branch/main/graph/badge.svg)](https://codecov.io/gh/stringertheory/clean-links) +[![Commit activity](https://img.shields.io/github/commit-activity/m/stringertheory/clean-links)](https://img.shields.io/github/commit-activity/m/stringertheory/clean-links) +[![License](https://img.shields.io/github/license/stringertheory/clean-links)](https://img.shields.io/github/license/stringertheory/clean-links) + +Tools for cleaning up linkss + +- **Github repository**: +- **Documentation** + +## Getting started with your project + +First, create a repository on GitHub with the same name as this project, and then run the following commands: + +```bash +git init -b main +git add . +git commit -m "init commit" +git remote add origin git@github.com:stringertheory/clean-links.git +git push -u origin main +``` + +Finally, install the environment and the pre-commit hooks with + +```bash +make install +``` + +You are now ready to start development on your project! +The CI/CD pipeline will be triggered when you open a pull request, merge to main, or when you create a new release. + +To finalize the set-up for publishing to PyPi or Artifactory, see [here](https://fpgmaas.github.io/cookiecutter-poetry/features/publishing/#set-up-for-pypi). +For activating the automatic documentation with MkDocs, see [here](https://fpgmaas.github.io/cookiecutter-poetry/features/mkdocs/#enabling-the-documentation-on-github). +To enable the code coverage reports, see [here](https://fpgmaas.github.io/cookiecutter-poetry/features/codecov/). + +## Releasing a new version + +- Create an API Token on [Pypi](https://pypi.org/). +- Add the API Token to your projects secrets with the name `PYPI_TOKEN` by visiting [this page](https://github.com/stringertheory/clean-links/settings/secrets/actions/new). +- Create a [new release](https://github.com/stringertheory/clean-links/releases/new) on Github. +- Create a new tag in the form `*.*.*`. + +For more details, see [here](https://fpgmaas.github.io/cookiecutter-poetry/features/cicd/#how-to-trigger-a-release). + +--- + +Repository initiated with [fpgmaas/cookiecutter-poetry](https://github.com/fpgmaas/cookiecutter-poetry). diff --git a/clean_links/clean.py b/clean_links/clean.py new file mode 100644 index 0000000..99a0536 --- /dev/null +++ b/clean_links/clean.py @@ -0,0 +1,101 @@ +import logging +import re +from urllib.parse import parse_qs, urlencode, urlsplit + +from clean_links.config import read_config +from clean_links.unshorten import unshorten_url + +clear_urls_rules = read_config() + + +def query_string(url: str, rules: list) -> str: + split = urlsplit(url) + params = parse_qs(split.query) + + delete_keys = {None, ""} + for rule in rules: + for key in params: + if re.match("^" + rule + "$", key, flags=re.IGNORECASE): + delete_keys.add(key) + + for delete_key in delete_keys: + params.pop(delete_key, "") # type: ignore[arg-type] + + params_string = urlencode(params, doseq=True) + + if params_string: + return split.path + "?" + params_string + else: + return split.path + + +def match_provider(provider: str, url: str, rules: dict) -> bool: + match_url = re.match(rules["urlPattern"], url) + match_exception = None + for exception_pattern in rules["exceptions"]: + try: + match_exception = re.match(exception_pattern, url) + except Exception: + logging.exception( + f"something's wrong with regex {exception_pattern!r} " + f"for provider {provider!r}." + ) + + if match_exception: + break + return bool(match_url and not match_exception) + + +def clear_url( + url: str, keep_query: bool = True, keep_fragment: bool = True +) -> str: + for provider_name, rules in clear_urls_rules["providers"].items(): + if match_provider(provider_name, url, rules): + for rule in rules["rawRules"]: + url = re.sub(rule, "", url, flags=re.IGNORECASE) + + split = urlsplit(url) + if keep_query: + full_path = query_string(url, rules["rules"]) + else: + full_path = split.path + + relative = full_path + if keep_fragment: + fragment_path = query_string(split.fragment, rules["rules"]) + if fragment_path: + relative += "#" + fragment_path + + url = f"{split.scheme}://{split.netloc}{relative}" + + return url + + +def main() -> None: + url = "https://www.amazon.com/Kobo-Glare-Free-Touchscreen-ComfortLight-Adjustable/dp/B0BCXLQNCC/ref=pd_ci_mcx_mh_mcx_views_0?pd_rd_w=Dx5dF&content-id=amzn1.sym.225b4624-972d-4629-9040-f1bf9923dd95%3Aamzn1.symc.40e6a10e-cbc4-4fa5-81e3-4435ff64d03b&pf_rd_p=225b4624-972d-4629-9040-f1bf9923dd95&pf_rd_r=A7JSDJGYR33BN5GRCV7V&pd_rd_wg=xW6Yf&pd_rd_r=4b8a3532-9e28-4857-a929-5e572d2c765f&pd_rd_i=B0BCXLQNCC" + + url = "https://trib.al/5m7fAg3" + # url = "https://tinyurl.com/yc2ft9m5" + # url = "https://bit.ly/3C4WXQ9" + # url = 'https://tinyurl.com/NewwAlemAndKibrom' + # url = "https://hubs.la/Q01HRjhm0" + # url = "https://buff.ly/3Omwkwd" + # url = "https://bit.ly/48RtRlw" + # url = "https://srv.buysellads.com/ads/long/x/TCHU7KSHTTTTTTH6NPRNPTTTTTTFNZMBKWTTTTTTA4RZC7VTTTTTTBZI5HINWLB6G3DIEMS4PABU5AIEQQY6BADG2HUT" + # url = "https://buff.ly/2RjYjMt" + + print(url) + print() + resolved = unshorten_url(url).get("resolved", "") + print(resolved) + print() + clear = clear_url(resolved) # , keep_query=False, keep_fragment=False) + print(clear) + # print(url) + # original, resolved, status = resolve_url(url, 10) + # print(original) + # print(resolved) + + +if __name__ == "__main__": + main() diff --git a/clean_links/clearurls.json b/clean_links/clearurls.json new file mode 100644 index 0000000..3840499 --- /dev/null +++ b/clean_links/clearurls.json @@ -0,0 +1,2559 @@ +{ + "providers": { + "amazon": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?amazon(?:\\.[a-z]{2,}){1,}", + "completeProvider": false, + "rules": [ + "p[fd]_rd_[a-z]*", + "qid", + "srs?", + "__mk_[a-z]{1,3}_[a-z]{1,3}", + "spIA", + "ms3_c", + "[a-z%0-9]*ie", + "refRID", + "colii?d", + "[^a-z%0-9]adId", + "qualifier", + "_encoding", + "smid", + "field-lbr_brands_browse-bin", + "ref_?", + "th", + "sprefix", + "crid", + "keywords", + "cv_ct_[a-z]+", + "linkCode", + "creativeASIN", + "ascsubtag", + "aaxitk", + "hsa_cr_id", + "sb-ci-[a-z]+", + "rnid", + "dchild", + "camp", + "creative", + "s", + "content-id" + ], + "referralMarketing": ["tag", "ascsubtag"], + "exceptions": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?amazon(?:\\.[a-z]{2,}){1,}\\/gp\\/.*?(?:redirector.html|cart\\/ajax-update.html|video\\/api\\/)", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?amazon(?:\\.[a-z]{2,}){1,}\\/(?:hz\\/reviews-render\\/ajax\\/|message-us\\?|s\\?)" + ], + "rawRules": ["\\/ref=[^/?]*"], + "redirections": [], + "forceRedirection": false + }, + "amazon search": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?amazon(?:\\.[a-z]{2,}){1,}\\/s\\?", + "completeProvider": false, + "rules": [ + "p[fd]_rd_[a-z]*", + "qid", + "srs?", + "__mk_[a-z]{1,3}_[a-z]{1,3}", + "spIA", + "ms3_c", + "[a-z%0-9]*ie", + "refRID", + "colii?d", + "[^a-z%0-9]adId", + "qualifier", + "_encoding", + "smid", + "field-lbr_brands_browse-bin", + "ref_?", + "th", + "sprefix", + "crid", + "cv_ct_[a-z]+", + "linkCode", + "creativeASIN", + "ascsubtag", + "aaxitk", + "hsa_cr_id", + "sb-ci-[a-z]+", + "rnid", + "dchild", + "camp", + "creative" + ], + "referralMarketing": ["tag"], + "rawRules": ["\\/ref=[^/?]*"], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "fls-na.amazon": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?fls-na\\.amazon(?:\\.[a-z]{2,}){1,}", + "completeProvider": true, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "google": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}", + "completeProvider": false, + "rules": [ + "ved", + "bi[a-z]*", + "gfe_[a-z]*", + "ei", + "source", + "gs_[a-z]*", + "site", + "oq", + "esrc", + "uact", + "cd", + "cad", + "gws_[a-z]*", + "atyp", + "vet", + "_u", + "je", + "dcr", + "ie", + "sei", + "sa", + "dpr", + "btn[a-z]*", + "usg", + "cd", + "cad", + "uact", + "aqs", + "sourceid", + "sxsrf", + "rlz", + "i-would-rather-use-firefox", + "pcampaignid", + "sca_esv" + ], + "referralMarketing": ["referrer"], + "rawRules": [], + "exceptions": [ + "^https?:\\/\\/mail\\.google\\.com\\/mail\\/u\\/", + "^https?:\\/\\/(?:docs|accounts)\\.google(?:\\.[a-z]{2,}){1,}", + "^https?:\\/\\/([a-z0-9-\\.])*(chat|drive)\\.google\\.com\\/videoplayback", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}(?:\\/upload)?\\/drive\\/", + "^https?:\\/\\/news\\.google\\.com.*\\?hl=.", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/s\\?tbm=map.*?gs_[a-z]*=.", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/(?:complete\\/search|setprefs|searchbyimage)", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/(?:appsactivity|aclk\\?)", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/safe[-]?browsing\\/([^&]+)" + ], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/url\\?.*?(?:url|q)=(https?[^&]+)", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/.*?adurl=([^&]+)", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/amp\\/s\\/([^&]+)" + ], + "forceRedirection": true + }, + "googleSearch": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/search\\?", + "completeProvider": false, + "rules": ["client", "sclient"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "googlesyndication": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?googlesyndication\\.com", + "completeProvider": true, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "doubleclick": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?doubleclick(?:\\.[a-z]{2,}){1,}", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?doubleclick(?:\\.[a-z]{2,}){1,}\\/.*?tag_for_child_directed_treatment=;%3F([^&]*)" + ], + "forceRedirection": false + }, + "googleadservices": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?googleadservices\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?googleadservices\\.com\\/.*?adurl=([^&]*)" + ], + "forceRedirection": false + }, + "globalRules": { + "urlPattern": ".*", + "completeProvider": false, + "rules": [ + "(?:%3F)?utm(?:_[a-z_]*)?", + "(?:%3F)?ga_[a-z_]+", + "(?:%3F)?yclid", + "(?:%3F)?_openstat", + "(?:%3F)?fb_action_(?:types|ids)", + "(?:%3F)?fb_(?:source|ref)", + "(?:%3F)?fbclid", + "(?:%3F)?action_(?:object|type|ref)_map", + "(?:%3F)?gs_l", + "(?:%3F)?mkt_tok", + "(?:%3F)?hmb_(?:campaign|medium|source)", + "(?:%3F)?gclid", + "(?:%3F)?otm_[a-z_]*", + "(?:%3F)?cmpid", + "(?:%3F)?os_ehash", + "(?:%3F)?_ga", + "(?:%3F)?_gl", + "(?:%3F)?__twitter_impression", + "(?:%3F)?wt_?z?mc", + "(?:%3F)?wtrid", + "(?:%3F)?[a-z]?mc", + "(?:%3F)?dclid", + "Echobox", + "(?:%3F)?spm", + "(?:%3F)?vn(?:_[a-z]*)+", + "(?:%3F)?tracking_source", + "(?:%3F)?ceneo_spo", + "(?:%3F)?itm_(?:campaign|medium|source)", + "(?:%3F)?__hsfp", + "(?:%3F)?__hssc", + "(?:%3F)?__hstc", + "(?:%3F)?_hsenc", + "(?:%3F)?__s", + "(?:%3F)?hsCtaTracking", + "(?:%3F)?mc_(?:eid|cid|tc)", + "(?:%3F)?ml_subscriber", + "(?:%3F)?ml_subscriber_hash", + "(?:%3F)?msclkid", + "(?:%3F)?oly_anon_id", + "(?:%3F)?oly_enc_id", + "(?:%3F)?rb_clickid", + "(?:%3F)?s_cid", + "(?:%3F)?vero_conv", + "(?:%3F)?vero_id", + "(?:%3F)?wickedid", + "(?:%3F)?twclid" + ], + "referralMarketing": ["(?:%3F)?ref_?", "(?:%3F)?referrer"], + "rawRules": [], + "exceptions": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?matrix\\.org\\/_matrix\\/", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?(?:cloudflare\\.com|prismic\\.io|tangerine\\.ca|gitlab\\.com)", + "^https?:\\/\\/myaccount.google(?:\\.[a-z]{2,}){1,}", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?gcsip\\.(?:com|nl)[^?]*\\?.*?&?ref_?=.", + "^https?:\\/\\/[^/]+/[^/]+/[^/]+\\/-\\/refs\\/switch[^?]*\\?.*?&?ref_?=.", + "^https?:\\/\\/bugtracker\\.[^/]*\\/[^?]+\\?.*?&?ref_?=[^/?&]*", + "^https?:\\/\\/comment-cdn\\.9gag\\.com\\/.*?comment-list.json\\?", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?battle\\.net\\/login", + "^https?:\\/\\/blizzard\\.com\\/oauth2", + "^https?:\\/\\/kreditkarten-banking\\.lbb\\.de", + "^https?:\\/\\/www\\.tinkoff\\.ru", + "^https?:\\/\\/www\\.cyberport\\.de\\/adscript\\.php", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?tweakers\\.net\\/ext\\/lt\\.dsp\\?.*?(?:%3F)?&?ref_?=.", + "^https?:\\/\\/git(lab)?\\.[^/]*\\/[^?]+\\?.*?&?ref_?=[^/?&]*", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?amazon(?:\\.[a-z]{2,}){1,}\\/message-us\\?", + "^https?:\\/\\/authorization\\.td\\.com", + "^https?:\\/\\/support\\.steampowered\\.com", + "^https?:\\/\\/privacy\\.vakmedianet\\.nl\\/.*?ref=", + "^https?:\\/\\/sso\\.serverplan\\.com\\/manage2fa\\/check\\?ref=", + "^https?:\\/\\/login\\.meijer\\.com\\/.*?\\?ref=", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?facebook\\.com\\/(?:login_alerts|ajax|should_add_browser)/", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?facebook\\.com\\/groups\\/member_bio\\/bio_dialog\\/", + "^https?:\\/\\/api\\.taiga\\.io", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?gog\\.com\\/click\\.html", + "^https?:\\/\\/login\\.progressive\\.com", + "^https?:\\/\\/www\\.sephora\\.com\\/api\\/", + "^https?:\\/\\/www\\.contestgirl\\.com", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?agenciatributaria\\.gob\\.es", + "^https?:\\/\\/login\\.ingbank\\.pl", + "^wss?:\\/\\/(?:[a-z0-9-]+\\.)*?zoom\\.us", + "^https?:\\/\\/api\\.bilibili\\.com", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?onet\\.pl\\/[^?]*\\?.*?utm_campaign=.", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?stripe\\.com\\/[^?]+.*?&?referrer=[^/?&]*", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?lichess\\.org\\/login.*?&?referrer=.*?", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?microsoft\\.com\\/.*?research\\/redirect", + "^https?:\\/\\/like.co\\/api\\/like\\/likebutton\\/[^?]+.*?&?referrer=[^/?&]*", + "^https?:\\/\\/button.like.co\\/in\\/.*?&?referrer=[^/?&]*", + "^https?:\\/\\/www\\.mma\\.go\\.kr", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?github\\.com", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?billiger\\.de\\/.*?mc=", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?\\.youtrack\\.cloud", + "^https?:\\/\\cu\\.bankid\\.com" + ], + "redirections": [], + "forceRedirection": false + }, + "adtech": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?adtech(?:\\.[a-z]{2,}){1,}", + "completeProvider": true, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "contentpass": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?contentpass\\.(?:net|de)", + "completeProvider": true, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "bf-ad": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?bf-ad(?:\\.[a-z]{2,}){1,}", + "completeProvider": true, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "amazon-adsystem": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?amazon-adsystem(?:\\.[a-z]{2,}){1,}", + "completeProvider": true, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?amazon-adsystem(?:\\.[a-z]{2,}){1,}\\/v3\\/oor\\?" + ], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?amazon-adsystem(?:\\.[a-z]{2,}){1,}\\/x\\/c\\/.+?\\/([^&]+)" + ], + "forceRedirection": false + }, + "adsensecustomsearchads": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?adsensecustomsearchads(?:\\.[a-z]{2,}){1,}", + "completeProvider": true, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "youtube": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?(youtube\\.com|youtu\\.be)", + "completeProvider": false, + "rules": ["feature", "gclid", "kw", "si"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?youtube\\.com\\/signin\\?.*?" + ], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?youtube\\.com\\/redirect?.*?q=([^&]*)" + ], + "forceRedirection": false + }, + "youtube_pagead": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?youtube\\.com\\/pagead", + "completeProvider": true, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "youtube_apiads": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?youtube\\.com\\/api\\/stats\\/ads", + "completeProvider": true, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "facebook": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?facebook\\.com", + "completeProvider": false, + "rules": [ + "hc_[a-z_%\\[\\]0-9]*", + "[a-z]*ref[a-z]*", + "__tn__", + "eid", + "__(?:xts|cft)__(?:\\[|%5B)\\d(?:\\]|%5D)", + "comment_tracking", + "dti", + "app", + "video_source", + "ftentidentifier", + "pageid", + "padding", + "ls_ref", + "action_history", + "tracking", + "referral_code", + "referral_story_type" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?facebook\\.com\\/.*?(plugins|ajax)\\/", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?facebook\\.com\\/dialog\\/(?:share|send)", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?facebook\\.com\\/groups\\/member_bio\\/bio_dialog\\/", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?facebook\\.com\\/photo\\.php\\?", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?facebook\\.com\\/privacy\\/specific_audience_selector_dialog\\/", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?facebook\\.com\\/photo\\/download\\/" + ], + "redirections": [ + "^https?:\\/\\/l[a-z]?\\.facebook\\.com/l\\.php\\?.*?u=(https?%3A%2F%2F[^&]*)" + ], + "forceRedirection": false + }, + "twitter": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?twitter.com", + "completeProvider": false, + "rules": ["(?:ref_?)?src", "s", "cn", "ref_url", "t"], + "referralMarketing": [], + "rawRules": [], + "exceptions": ["^https?:\\/\\/twitter.com\\/i\\/redirect"], + "redirections": [], + "forceRedirection": false + }, + "x": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?x.com", + "completeProvider": false, + "rules": ["(?:ref_?)?src", "s", "cn", "ref_url", "t"], + "referralMarketing": [], + "rawRules": [], + "exceptions": ["^https?:\\/\\/x.com\\/i\\/redirect"], + "redirections": [], + "forceRedirection": false + }, + "reddit": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?reddit.com", + "completeProvider": false, + "rules": [ + "%24deep_link", + "\\$deep_link", + "correlation_id", + "ref_campaign", + "ref_source", + "%243p", + "rdt", + "\\$3p", + "%24original_url", + "\\$original_url", + "_branch_match_id" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/out\\.reddit\\.com\\/.*?url=([^&]*)", + "^https?:\\/\\/click\\.redditmail\\.com\\/.*?url=([^&]*)" + ], + "forceRedirection": false + }, + "netflix": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?netflix.com", + "completeProvider": false, + "rules": ["trackId", "tctx", "jb[a-z]*?"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "techcrunch": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?techcrunch\\.com", + "completeProvider": false, + "rules": ["ncid", "sr", "sr_share"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "bing": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?bing(?:\\.[a-z]{2,}){1,}", + "completeProvider": false, + "rules": ["cvid", "form", "sk", "sp", "sc", "qs", "qp"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?bing(?:\\.[a-z]{2,}){1,}\\/WS\\/redirect\\/" + ], + "redirections": [], + "forceRedirection": false + }, + "tweakers": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?tweakers\\.net", + "completeProvider": false, + "rules": ["nb", "u"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "twitch": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?twitch\\.com", + "completeProvider": false, + "rules": ["tt_medium", "tt_content"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "vivaldi": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?vivaldi\\.com", + "completeProvider": false, + "rules": ["pk_campaign", "pk_kwd"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "indeed": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?indeed\\.com", + "completeProvider": false, + "rules": ["from", "alid", "[a-z]*tk"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?indeed\\.com\\/rc\\/clk" + ], + "redirections": [], + "forceRedirection": false + }, + "hhdotru": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?hh\\.ru", + "completeProvider": false, + "rules": ["vss", "t", "swnt", "grpos", "ptl", "stl", "exp", "plim"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "ebay": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?ebay(?:\\.[a-z]{2,}){1,}", + "completeProvider": false, + "rules": ["_trkparms", "_trksid", "_from", "hash"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?rover\\.ebay(?:\\.[a-z]{2,}){1,}\\/rover.*mpre=([^&]*)" + ], + "forceRedirection": false + }, + "cnet": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?cnet\\.com", + "completeProvider": false, + "rules": ["ftag"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "imdb.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?imdb\\.com", + "completeProvider": false, + "rules": ["ref_", "pf_rd_[a-z]*"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "govdelivery.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?govdelivery\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?links\\.govdelivery\\.com.*\\/track\\?.*(https?:\\/\\/.*)" + ], + "forceRedirection": false + }, + "walmart.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?walmart\\.com", + "completeProvider": false, + "rules": ["u1", "ath[a-z]*"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "net-parade.it": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?net\\-parade\\.it", + "completeProvider": false, + "rules": ["pl"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "prvnizpravy.cz": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?prvnizpravy\\.cz", + "completeProvider": false, + "rules": ["xid"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "youku.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?youku\\.com", + "completeProvider": false, + "rules": ["tpa"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "nytimes.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?nytimes\\.com", + "completeProvider": false, + "rules": ["smid"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "tchibo.de": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?tchibo\\.de", + "completeProvider": false, + "rules": ["wbdcd"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "steampowered": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?steampowered\\.com", + "completeProvider": false, + "rules": ["snr"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "steamcommunity": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?steamcommunity\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?steamcommunity\\.com\\/linkfilter\\/\\?url=([^&]*)" + ], + "forceRedirection": false + }, + "mozaws.net": { + "urlPattern": "https?:\\/\\/outgoing\\.prod\\.mozaws\\.net\\/", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": ["https?:\\/\\/[^/]+\\/v1\\/[0-9a-f]{64}\\/(.*)"], + "forceRedirection": false + }, + "shutterstock.com": { + "urlPattern": "https?:\\/\\/([a-z0-9-.]*\\.)shutterstock\\.com", + "completeProvider": false, + "rules": ["src"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "mozilla.org": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?mozilla\\.org", + "completeProvider": false, + "rules": ["src", "platform", "redirect_source"], + "referralMarketing": [], + "rawRules": [], + "exceptions": ["^https?:\\/\\/(?:[a-z0-9-]+\\.)*?mozilla.org\\/api"], + "redirections": [], + "forceRedirection": false + }, + "readdc.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?readdc\\.com", + "completeProvider": false, + "rules": ["ref"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "dailycodingproblem.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?dailycodingproblem\\.com", + "completeProvider": false, + "rules": ["email"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "github.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?github\\.com", + "completeProvider": false, + "rules": ["email_token", "email_source"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "deviantart.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?deviantart\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?deviantart\\.com\\/.*?\\/outgoing\\?(.*)" + ], + "forceRedirection": false + }, + "site2.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?site2\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?site2\\.com.*?\\?.*=(.*)" + ], + "forceRedirection": false + }, + "site.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?site\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?site\\.com.*?\\?to=([^&]*)" + ], + "forceRedirection": false + }, + "site3.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?site3\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?site3\\.com.*?\\?r=([^&]*)" + ], + "forceRedirection": false + }, + "aliexpress": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?aliexpress(?:\\.[a-z]{2,}){1,}", + "completeProvider": false, + "rules": [ + "ws_ab_test", + "btsid", + "algo_expid", + "algo_pvid", + "gps-id", + "scm[_a-z-]*", + "cv", + "af", + "mall_affr", + "sk", + "dp", + "terminal_id", + "aff_request_id" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "mozillazine.org": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?mozillazine\\.org", + "completeProvider": false, + "rules": ["sid"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "9gag.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?9gag\\.com", + "completeProvider": false, + "rules": ["ref"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [ + "^https?:\\/\\/comment-cdn\\.9gag\\.com\\/.*?comment-list.json\\?" + ], + "redirections": [], + "forceRedirection": false + }, + "linksynergy.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?linksynergy\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?linksynergy\\.com\\/.*?murl=([^&]*)" + ], + "forceRedirection": false + }, + "giphy.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?giphy\\.com", + "completeProvider": false, + "rules": ["ref"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "gate.sc": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?gate\\.sc", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?gate\\.sc\\/.*?url=([^&]*)" + ], + "forceRedirection": false + }, + "vk.com": { + "urlPattern": "^https?:\\/\\/vk\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": ["^https?:\\/\\/vk\\.com\\/away\\.php\\?to=([^&]*)"], + "forceRedirection": false + }, + "woot.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?woot\\.com", + "completeProvider": false, + "rules": ["ref_?"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "vitamix.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?vitamix\\.com", + "completeProvider": false, + "rules": ["_requestid", "cid", "dl", "di", "sd", "bi"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "curseforge.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?curseforge\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?curseforge\\.com\\/linkout\\?remoteUrl=([^&]*)" + ], + "forceRedirection": false + }, + "messenger.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?messenger\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/l\\.messenger\\.com\\/l\\.php\\?u=([^&]*)" + ], + "forceRedirection": false + }, + "nypost.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?nypost\\.com", + "completeProvider": false, + "rules": ["__twitter_impression"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "ozon.ru": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?ozon\\.ru", + "completeProvider": false, + "rules": ["partner"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "norml.org": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?norml\\.org", + "completeProvider": false, + "rules": [ + "link_id", + "can_id", + "source", + "email_referrer", + "email_subject" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "LinkedIn": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?linkedin\\.com", + "completeProvider": false, + "rules": ["refId", "trk", "li[a-z]{2}", "trackingId"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "LinkedIn Learning": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?linkedin\\.com\\/learning", + "completeProvider": false, + "rules": ["u"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "smartredirect.de": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?smartredirect\\.de", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?smartredirect\\.de.*?url=([^&]*)" + ], + "forceRedirection": false + }, + "SPIEGEL ONLINE": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?spiegel\\.de", + "completeProvider": false, + "rules": ["b"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "rutracker.org": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?rutracker\\.org", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [".*url=([^&]*)"], + "forceRedirection": false + }, + "instagram": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?instagram\\.com", + "completeProvider": false, + "rules": ["igshid"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [".*u=([^&]*)"], + "forceRedirection": false + }, + "lazada.com.my": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?lazada\\.com\\.my", + "completeProvider": false, + "rules": ["ad_src", "did", "pa", "mp", "impsrc", "cid", "pos"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "imgsrc.ru": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?imgsrc\\.ru", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?dlp\\.imgsrc\\.ru\\/go\\/\\d+\\/\\d+\\/\\d+\\/([^&]*)" + ], + "forceRedirection": false + }, + "boredpanda.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?boredpanda\\.com", + "completeProvider": false, + "rules": ["h"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "awstrack.me": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?awstrack\\.me", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?awstrack\\.me\\/.*\\/(https?.*?)\\/" + ], + "forceRedirection": false + }, + "exactag.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?exactag\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?exactag\\.com.*url=([^&]*)" + ], + "forceRedirection": false + }, + "bahn.de": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?bahn\\.de", + "completeProvider": false, + "rules": ["dbkanal_[0-9]{3}"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "disq.us": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?disq\\.us", + "completeProvider": false, + "rules": ["cuid"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?disq\\.us\\/.*?url=([^&]*)%3A" + ], + "forceRedirection": false + }, + "anonym.to": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?anonym\\.to", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?anonym\\.to.*\\?([^&]*)" + ], + "forceRedirection": false + }, + "moosejaw.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?moosejaw\\.com", + "completeProvider": false, + "rules": [ + "cm_lm", + "cm_mmc", + "webUserId", + "spMailingID", + "spUserID", + "spJobID", + "spReportId" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "spotify.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?spotify\\.com", + "completeProvider": false, + "rules": ["si"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "yandex": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?(?:yandex(?:\\.[a-z]{2,}){1,}|ya\\.ru)", + "completeProvider": false, + "rules": ["lr", "redircnt"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "healio.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?healio\\.com", + "completeProvider": false, + "rules": ["ecp", "m_bt"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "zoho.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?zoho\\.com", + "completeProvider": false, + "rules": ["iref"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "snapchat.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?snapchat\\.com", + "completeProvider": false, + "rules": ["sc_referrer", "sc_ua"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "medium.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?medium\\.com", + "completeProvider": false, + "rules": ["source"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "swp.de": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?swp\\.de", + "completeProvider": false, + "rules": ["source"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "wps.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?wps\\.com", + "completeProvider": false, + "rules": ["from"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "accounts.firefox.com": { + "urlPattern": "^https?:\\/\\/(?:accounts\\.)?firefox\\.com", + "completeProvider": false, + "rules": ["context", "entrypoint", "form_type"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "support.mozilla.org": { + "urlPattern": "^https?:\\/\\/(?:support\\.)?mozilla\\.org", + "completeProvider": false, + "rules": ["as"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "ClearURLsTest": { + "urlPattern": "^https?:\\/\\/kevinroebert\\.gitlab\\.io\\/ClearUrls\\/void\\/index\\.html", + "completeProvider": false, + "rules": ["test"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/kevinroebert\\.gitlab\\.io\\/ClearUrls\\/void\\/index\\.html\\?url=([^&]*)" + ], + "forceRedirection": false + }, + "ClearURLsTestBlock": { + "urlPattern": "^https?:\\/\\/kevinroebert\\.gitlab\\.io\\/ClearUrls\\/void\\/block\\.svg", + "completeProvider": true, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "ClearURLsTest2": { + "urlPattern": "^https?:\\/\\/test\\.clearurls\\.xyz\\/void\\/index\\.html", + "completeProvider": false, + "rules": ["test"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/test\\.clearurls\\.xyz\\/void\\/index\\.html\\?url=([^&]*)" + ], + "forceRedirection": false + }, + "ClearURLsTestBlock2": { + "urlPattern": "^https?:\\/\\/test\\.clearurls\\.xyz\\/void\\/block\\.svg", + "completeProvider": true, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "diepresse.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?diepresse\\.com", + "completeProvider": false, + "rules": ["from", "xtor", "xt_at"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "newsletter.lidl.com": { + "urlPattern": "^https?:\\/\\/newsletter\\.lidl(?:\\.[a-z]{2,}){1,}", + "completeProvider": false, + "rules": ["x"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "allegro.pl": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?allegro\\.pl", + "completeProvider": false, + "rules": ["reco_id", "sid"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "backcountry.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?backcountry\\.com", + "completeProvider": false, + "rules": [ + "CMP_SKU", + "MER", + "mr:trackingCode", + "mr:device", + "mr:adType", + "iv_", + "CMP_ID", + "k_clickid", + "rmatt", + "INT_ID", + "ti", + "fl" + ], + "referralMarketing": ["mr:referralID"], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "meetup.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?meetup\\.com", + "completeProvider": false, + "rules": ["rv", "_xtd"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "apple.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?apple\\.com", + "completeProvider": false, + "rules": ["app", "ign-itsc[a-z]+"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "alabout.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?alabout\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?alabout\\.com.*url=([^&]*)" + ], + "forceRedirection": false + }, + "newyorker.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?newyorker\\.com", + "completeProvider": false, + "rules": ["source", "bxid", "cndid", "esrc", "mbid"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "gog.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?gog\\.com", + "completeProvider": false, + "rules": ["track_click", "link_id"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "tradedoubler.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?tradedoubler\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?tradedoubler\\.com.*(?:url|_td_deeplink)=([^&]*)" + ], + "forceRedirection": false + }, + "theguardian.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?theguardian\\.com", + "completeProvider": false, + "rules": ["CMP"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "srvtrck.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?srvtrck\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?srvtrck\\.com.*url=([^&]*)" + ], + "forceRedirection": false + }, + "mysku.ru": { + "urlPattern": "^https?:\\/\\/mysku\\.ru", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": ["^https?:\\/\\/mysku\\.ru.*r=([^&]*)"], + "forceRedirection": false + }, + "admitad.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?admitad\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?admitad\\.com.*ulp=([^&]*)" + ], + "forceRedirection": false + }, + "taobao.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?taobao\\.com", + "completeProvider": false, + "rules": [ + "price", + "sourceType", + "suid", + "ut_sk", + "un", + "share_crt_v", + "sp_tk", + "cpp", + "shareurl", + "short_name", + "app", + "scm[_a-z-]*", + "pvid", + "algo_expid", + "algo_pvid", + "ns", + "abbucket", + "ali_refid", + "ali_trackid", + "acm", + "utparam", + "pos", + "abtest", + "trackInfo", + "utkn", + "scene", + "mytmenu", + "turing_bucket", + "lygClk", + "impid", + "bftTag", + "bftRwd", + "spm", + "_u" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "tmall.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?tmall\\.com", + "completeProvider": false, + "rules": [ + "price", + "sourceType", + "suid", + "ut_sk", + "un", + "share_crt_v", + "sp_tk", + "cpp", + "shareurl", + "short_name", + "app", + "scm[_a-z-]*", + "pvid", + "algo_expid", + "algo_pvid", + "ns", + "abbucket", + "ali_refid", + "ali_trackid", + "acm", + "utparam", + "pos", + "abtest", + "trackInfo", + "user_number_id", + "utkn", + "scene", + "mytmenu", + "turing_bucket", + "lygClk", + "impid", + "bftTag", + "bftRwd", + "activity_id" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "tb.cn": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?tb\\.cn", + "completeProvider": false, + "rules": ["sm"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "bilibili.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?bilibili\\.com", + "completeProvider": false, + "rules": [ + "callback", + "spm_id_from", + "from_source", + "from", + "seid", + "mid", + "share_source", + "msource", + "refer_from", + "share_from", + "share_medium", + "share_source", + "share_plat", + "share_tag", + "share_session_id", + "timestamp", + "unique_k", + "vd_source", + "plat_id", + "buvid", + "is_story_h5", + "up_id" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [ + "^https?:\\/\\/api\\.bilibili\\.com", + "^https?:\\/\\/space\\.bilibili\\.com" + ], + "redirections": [], + "forceRedirection": false + }, + "m.bilibili.com": { + "urlPattern": "^https?:\\/\\/m\\.bilibili\\.com", + "completeProvider": false, + "rules": ["bbid", "ts"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "live.bilibili.com": { + "urlPattern": "^https?:\\/\\/live\\.bilibili\\.com", + "completeProvider": false, + "rules": ["visit_id", "session_id", "broadcast_type", "is_room_feed"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "marketscreener.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?marketscreener\\.com", + "completeProvider": false, + "rules": [ + "type_recherche", + "mots", + "noredirect", + "RewriteLast", + "lien", + "aComposeInputSearch", + "type_recherche_forum", + "add_mots", + "countview" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?marketscreener\\.com\\/search\\/\\?" + ], + "redirections": [], + "forceRedirection": false + }, + "marketscreener.com search": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?marketscreener\\.com\\/search\\/\\?", + "completeProvider": false, + "rules": [ + "type_recherche", + "noredirect", + "RewriteLast", + "lien", + "aComposeInputSearch", + "type_recherche_forum", + "countview" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "bestbuy.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?bestbuy\\.com", + "completeProvider": false, + "rules": ["irclickid", "irgwc", "loc", "acampID", "mpid", "intl"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "digidip.net": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?digidip\\.net", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?digidip\\.net.*url=([^&]*)" + ], + "forceRedirection": false + }, + "tiktok.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?tiktok\\.com", + "completeProvider": false, + "rules": [ + "u_code", + "preview_pb", + "_d", + "timestamp", + "user_id", + "share_app_name", + "share_iid", + "source" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "autoplus.fr": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?autoplus\\.fr", + "completeProvider": false, + "rules": ["idprob", "hash", "sending_id", "site_id", "dr_tracker"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "bigfishgames.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?bigfishgames\\.com", + "completeProvider": false, + "rules": ["pc", "npc", "npv[0-9]+", "npi"], + "referralMarketing": [], + "rawRules": ["\\?pc$"], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "dpbolvw.net": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?dpbolvw\\.net", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?dpbolvw\\.net.*url=([^&]*)" + ], + "forceRedirection": false + }, + "humblebundle.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?humblebundle\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": ["partner"], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "cafepedagogique.net": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?cafepedagogique\\.net", + "completeProvider": false, + "rules": ["actId", "actCampaignType", "actSource"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "bloculus.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?bloculus\\.com", + "completeProvider": false, + "rules": ["tl_[a-z_]+"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "mailpanion.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?mailpanion\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?mailpanion\\.com.*destination=([^&]*)" + ], + "forceRedirection": false + }, + "signtr.website": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?signtr\\.website", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?signtr\\.website.*redirect=([^&]*)" + ], + "forceRedirection": false + }, + "mailtrack.io": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?mailtrack\\.io", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?mailtrack\\.io.*url=([^&]*)" + ], + "forceRedirection": false + }, + "zillow.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?zillow\\.com", + "completeProvider": false, + "rules": ["rtoken"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "realtor.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?realtor\\.com", + "completeProvider": false, + "rules": ["ex", "identityID", "MID", "RID"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "redfin.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?redfin\\.com", + "completeProvider": false, + "rules": ["riftinfo"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "epicgames.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?epicgames\\.com", + "completeProvider": false, + "rules": ["epic_affiliate", "epic_gameId"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "onet.pl": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?onet\\.pl", + "completeProvider": false, + "rules": ["srcc", "utm_v", "utm_medium", "utm_source"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "allrecipes.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?allrecipes\\.com", + "completeProvider": false, + "rules": [ + "internalSource", + "referringId", + "referringContentType", + "clickId" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "europe1.fr": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?europe1\\.fr", + "completeProvider": false, + "rules": ["xtor"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "effiliation.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?effiliation\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?effiliation\\.com.*url=([^&]*)" + ], + "forceRedirection": false + }, + "argos.co.uk": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?argos\\.co\\.uk", + "completeProvider": false, + "rules": [ + "istCompanyId", + "istFeedId", + "istItemId", + "istBid", + "clickOrigin" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "hlserve.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?hlserve\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?hlserve\\.com.*dest=([^&]*)" + ], + "forceRedirection": false + }, + "thunderbird.net": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?thunderbird\\.net", + "completeProvider": false, + "rules": ["src"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "cnbc.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?cnbc\\.com", + "completeProvider": false, + "rules": ["__source"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "roblox.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?roblox\\.com", + "completeProvider": false, + "rules": ["refPageId"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "cell.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?cell\\.com", + "completeProvider": false, + "rules": ["_returnURL"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "academic.oup.com": { + "urlPattern": "^https?:\\/\\/academic\\.oup\\.com", + "completeProvider": false, + "rules": ["redirectedFrom"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "flexlinkspro.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?flexlinkspro\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?flexlinkspro\\.com.*url=([^&]*)" + ], + "forceRedirection": false + }, + "agata88.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?agata88\\.com", + "completeProvider": false, + "rules": ["source"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "hs.fi": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?hs\\.fi", + "completeProvider": false, + "rules": ["share"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "yle.fi": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?yle\\.fi", + "completeProvider": false, + "rules": ["origin"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "ccbill.com": { + "urlPattern": "^https?:\\/\\/refer\\.ccbill\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": ["^https?:\\/\\/refer\\.ccbill\\.com.*HTML=([^&]*)"], + "forceRedirection": false + }, + "flipkart": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?flipkart\\.com", + "completeProvider": false, + "rules": [ + "otracker.?", + "ssid", + "[cilp]id", + "marketplace", + "store", + "srno", + "store", + "ppn", + "ppt", + "fm", + "collection-tab-name", + "sattr\\[\\]", + "p\\[\\]", + "st" + ], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "idealo.de": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?idealo\\.de", + "completeProvider": false, + "rules": [ + "sid", + "src", + "siteId", + "lcb", + "leadOutUrl", + "offerListId", + "osId", + "cancelUrl", + "disc" + ], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "idealo-partner.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?idealo-partner\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?idealo-partner\\.com.*trg=([^&]*)" + ], + "forceRedirection": false + }, + "teletrader.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?teletrader\\.com", + "completeProvider": false, + "rules": ["internal"], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "webgains.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?webgains\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?webgains\\.com.*wgtarget=([^&]*)" + ], + "forceRedirection": false + }, + "deeplearning.ai": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?deeplearning\\.ai", + "completeProvider": false, + "rules": ["ecid", "_hsmi", "_hsenc"], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "getpocket.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?getpocket\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?getpocket\\.com.*url=([^&]*)" + ], + "forceRedirection": false + }, + "gamespot.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?gamespot\\.com", + "completeProvider": false, + "rules": ["PostType", "ServiceType", "ftag", "UniqueID", "TheTime"], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "tokopedia.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?tokopedia\\.com", + "completeProvider": false, + "rules": ["src", "trkid", "whid"], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [ + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?tokopedia\\.com\\/promo.*r=([^&]*)" + ], + "forceRedirection": false + }, + "wkorea.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?wkorea\\.com", + "completeProvider": false, + "rules": ["ddw", "ds_ch"], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "eonline.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?eonline\\.com", + "completeProvider": false, + "rules": ["source", "medium", "content"], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "reuters.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?reuters\\.com", + "completeProvider": false, + "rules": ["taid"], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "app.adjust.com": { + "urlPattern": "^https?:\\/\\/app\\.adjust\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": ["^https?:\\/\\/app\\.adjust\\.com.*redirect=([^&]*)"], + "forceRedirection": false + }, + "change.org": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?change\\.org", + "completeProvider": false, + "rules": ["source_location", "psf_variant", "share_intent"], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "ceneo.pl": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?ceneo\\.pl", + "completeProvider": false, + "rules": ["tag"], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "wired.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?wired\\.com", + "completeProvider": false, + "rules": ["intcid"], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "alibaba cloud arms": { + "urlPattern": "^https?:\\/\\/arms-retcode\\.aliyuncs\\.com", + "completeProvider": false, + "rules": [ + "pid", + "uid", + "tag", + "release", + "environment", + "sample", + "behavior", + "enableSPA", + "enableLinkTrace", + "page", + "begin", + "c2", + "c3", + "success", + "code", + "msg", + "api", + "traceId", + "pv_id", + "flag", + "sr", + "vp", + "ct", + "_v", + "sampling", + "dl", + "post_res" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "nikkei": { + "urlPattern": "^https?://(?:[a-z0-9-]+\\.)*?nikkei\\.co(?:m|\\.jp)", + "completeProvider": false, + "rules": ["adid", "i_cid", "n_cid", "waad"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "weibo": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?weibo\\.(cn|com)", + "completeProvider": false, + "rules": ["weibo_id", "dt_dapp"], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": false + }, + "fiverr.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?fiverr\\.com", + "completeProvider": false, + "rules": ["context_referrer", "source", "ref_ctx_id", "funnel"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "etsy.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?etsy\\.com", + "completeProvider": false, + "rules": ["click_key", "click_sum", "organic_search_click"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "magento.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?magento\\.com", + "completeProvider": false, + "rules": ["itm_campaign", "itm_medium", "itm_source", "itm_term"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "novinky.cz": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?novinky\\.cz", + "completeProvider": false, + "rules": [ + "dop_ab_variant", + "dop_source_zone_name", + "dop_req_id", + "dop_id", + "source", + "seq_no" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "aktualne.cz": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?aktualne\\.cz", + "completeProvider": false, + "rules": [ + "dop_ab_variant", + "dop_source_zone_name", + "dop_req_id", + "dop_id" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "seznamzpravy.cz": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?seznamzpravy\\.cz", + "completeProvider": false, + "rules": [ + "dop_ab_variant", + "dop_source_zone_name", + "dop_req_id", + "dop_id", + "source", + "seq_no" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "billiger.de": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?billiger\\.de", + "completeProvider": false, + "rules": ["log", "p"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "respekt.cz": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?respekt\\.cz", + "completeProvider": false, + "rules": [ + "sznclid", + "dop_ab_variant", + "dop_source_zone_name", + "dop_req_id", + "dop_id" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "faei.cz": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?faei\\.cz", + "completeProvider": false, + "rules": [ + "sznclid", + "dop_ab_variant", + "dop_source_zone_name", + "dop_req_id", + "dop_id" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "iprima.cz": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?iprima\\.cz", + "completeProvider": false, + "rules": [ + "sznclid", + "dop_ab_variant", + "dop_source_zone_name", + "dop_req_id", + "dop_id" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "nova.cz": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?nova\\.cz", + "completeProvider": false, + "rules": [ + "sznclid", + "dop_ab_variant", + "dop_source_zone_name", + "dop_req_id", + "dop_id" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "duckduckgo": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?duckduckgo\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": ["^https?:\\/\\/duckduckgo\\.com\\/l\\/.*?uddg=([^&]+)"], + "forceRedirection": false + }, + "mercadolibre": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?mercadolibre\\.com", + "completeProvider": false, + "rules": [ + "DEAL_ID", + "L", + "S", + "T", + "V", + "pdp_filters", + "position", + "search_layout", + "tracking_id", + "type", + "c_[_a-zA-Z]+", + "me\\.[_a-zA-Z]+", + "reco_[_a-zA-Z]+" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "quizlet": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?quizlet\\.com", + "completeProvider": false, + "rules": ["funnelUUID"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "bbc": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?bbc\\.com", + "completeProvider": false, + "rules": ["xtor", "at_[a-z_]+"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "airbnb": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?airbnb\\.(com|ae|ca|co\\.in|co\\.nz|co\\.uk|co\\.za|com\\.au|com\\.mt|com\\.sg|de|gy|ie)", + "completeProvider": false, + "rules": [ + "federated_search_id", + "search_type", + "source", + "source_impression_id" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "partner-ads.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?partner-ads\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/.*?partner-ads\\.com\\/.*?htmlurl=([^&]+)" + ], + "forceRedirection": false + }, + "kahoot.it": { + "urlPattern": "^https?://(?:[a-z0-9-]+\\.)*?kahoot\\.it", + "completeProvider": false, + "rules": ["refer_method"], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "href.li": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?href\\.li", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": ["^https?:\\/\\/href\\.li\\/\\?(http.+)"], + "forceRedirection": false + }, + "adform.net": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?adform\\.net", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/track\\.adform\\.net\\/C\\/.*?ckurl=([^&]+)" + ], + "forceRedirection": false + }, + "artefact.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?artefact\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/.*?artefact\\.com\\/trck\\/.*?deeplinkurl=([^&]+)" + ], + "forceRedirection": false + }, + "awin1.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?awin1\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": ["^https?:\\/\\/.*?awin1\\.com\\/.*?ued=([^&]+)"], + "forceRedirection": false + }, + "telekom.de": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?telekom\\.de", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/aaa\\.telekom\\.de\\/trck\\/.*?deeplinkurl=([^&]+)" + ], + "forceRedirection": false + }, + "cc.loginfra.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?loginfra\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": ["^https?:\\/\\/cc\\.loginfra\\.com\\/.*?u=([^&]+)"], + "forceRedirection": false + }, + "t.umblr.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?umblr\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": ["^https?:\\/\\/t\\.umblr\\.com\\/redirect\\?z=([^&]+)"], + "forceRedirection": false + } + } +} diff --git a/clean_links/clearurls_config.json b/clean_links/clearurls_config.json new file mode 100644 index 0000000..0e522bf --- /dev/null +++ b/clean_links/clearurls_config.json @@ -0,0 +1,17 @@ +{ + "providers": { + "globalRules": { + "rules": ["(?:%3F)?utm_(.*)?"] + }, + "wired.com": { + "rules": ["mbid"] + }, + "google": { + "rules": ["iflsig"] + }, + "bloomberg": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?bloomberg\\.com", + "rules": ["cmpid="] + } + } +} diff --git a/clean_links/config.py b/clean_links/config.py new file mode 100644 index 0000000..93df1e1 --- /dev/null +++ b/clean_links/config.py @@ -0,0 +1,34 @@ +import json + + +def read_config( + additional_config_filename: str = "clearurls_config.json" +) -> dict: + with open("clearurls.json") as infile: + clear_urls_rules = dict(json.load(infile)) + + with open(additional_config_filename) as infile: + additional_rules = json.load(infile) + + for provider_name, rules in additional_rules["providers"].items(): + provider = clear_urls_rules["providers"].get(provider_name) + if not provider: + clear_urls_rules["providers"][provider_name] = { + "urlPattern": None, + "completeProvider": False, + "rules": [], + "referralMarketing": [], + "exceptions": [], + "rawRules": [], + "redirections": [], + "forceRedirection": False, + } + provider = clear_urls_rules["providers"][provider_name] + + for key, value in rules.items(): + if key in ["rules"]: + provider[key].extend(value) + else: + provider[key] = value + + return clear_urls_rules diff --git a/clean_links/unshorten.py b/clean_links/unshorten.py new file mode 100644 index 0000000..6fa7bbe --- /dev/null +++ b/clean_links/unshorten.py @@ -0,0 +1,85 @@ +import logging + +import requests +from urllib3.exceptions import InsecureRequestWarning + +# suppress warning about that's given when using verify=False +requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) + +__packagename__ = "CleanLinks" +__version__ = "0.01" +__url__ = "https://github.com/stringertheory/clean-links" + +USER_AGENT = f"{__packagename__}/{__version__} ({__url__})" +USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) +HEADERS = { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "accept-encoding": "gzip, deflate, br", + "accept-language": "en-US,en;q=0.9", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "same-origin", + "sec-fetch-user": "?1", + "upgrade-insecure-requests": "1", + "user-agent": USER_AGENT, +} + + +def get_last_url_from_exception(exc: Exception) -> str | None: + result = None + + try: + if exc.response and exc.response.url: + result = exc.response.url + elif exc.request: + result = exc.request.url + except Exception as exc: + logging.exception("exception occurred while getting last url") + + return result + + +def unshorten_url( + url: str, timeout: int = 9, verify: bool = False, headers: dict = HEADERS +) -> dict: + with requests.Session() as session: + try: + response = session.head( + url, + allow_redirects=True, + timeout=timeout, + headers=headers, + verify=verify, + ) + except Exception as exc: + return { + "url": url, + "resolved": get_last_url_from_exception(exc), + "status": None, + "exception": f"{type(exc).__name__}: {exc}", + } + else: + return { + "url": url, + "resolved": response.url, + "status": response.status_code, + "exception": None, + } + + +def main() -> None: + url = "https://trib.al/5m7fAg3" + url = "https://www.bloomberg.com/news/articles/2024-01-24/cryptocurrency-ai-electricity-demand-seen-doubling-in-three-years?cmpid%3D=socialflow-twitter-tech&utm_content=tech&utm_medium=social&utm_campaign=socialflow-organic&utm_source=twitter" + url = "https://tinyurl.com/yc2ft9m5" + url = "https://bit.ly/3C4WXQ9" + # url = 'https://tinyurl.com/NewwAlemAndKibrom' + url = "https://hubs.la/Q01HRjhm0" + + print(unshorten_url(url)) + + +if __name__ == "__main__": + main() diff --git a/poetry.lock b/poetry.lock index 391c00f..3ca75fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1152,6 +1152,20 @@ virtualenv = ">=20.25" docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-argparse-cli (>=1.11.1)", "sphinx-autodoc-typehints (>=1.25.2)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.11)"] testing = ["build[virtualenv] (>=1.0.3)", "covdefaults (>=2.3)", "detect-test-pollution (>=1.2)", "devpi-process (>=1)", "diff-cover (>=8.0.2)", "distlib (>=0.3.8)", "flaky (>=3.7)", "hatch-vcs (>=0.4)", "hatchling (>=1.21)", "psutil (>=5.9.7)", "pytest (>=7.4.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-xdist (>=3.5)", "re-assert (>=1.1)", "time-machine (>=2.13)", "wheel (>=0.42)"] +[[package]] +name = "types-requests" +version = "2.31.0.20240125" +description = "Typing stubs for requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-requests-2.31.0.20240125.tar.gz", hash = "sha256:03a28ce1d7cd54199148e043b2079cdded22d6795d19a2c2a6791a4b2b5e2eb5"}, + {file = "types_requests-2.31.0.20240125-py3-none-any.whl", hash = "sha256:9592a9a4cb92d6d75d9b491a41477272b710e021011a2a3061157e2fb1f1a5d1"}, +] + +[package.dependencies] +urllib3 = ">=2" + [[package]] name = "typing-extensions" version = "4.9.0" @@ -1256,4 +1270,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<4.0" -content-hash = "be0e72228d5e1cf1a8e202fd33c7e615bdb5e055968e07ddf44b95a9f28eb9c6" +content-hash = "63f66ed2c750ea88a2cc9204bdae2ad62f78862dd50bd22eca0ac91e5bd5bf15" diff --git a/pyproject.toml b/pyproject.toml index 12fc4f5..9fa0e8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ pytest-cov = "^4.0.0" mypy = "^1.5.1" pre-commit = "^3.4.0" tox = "^4.11.1" +types-requests = "^2.31.0.20240125" [tool.poetry.group.docs.dependencies] mkdocs = "^1.4.2" @@ -38,13 +39,14 @@ check_untyped_defs = "True" warn_return_any = "True" warn_unused_ignores = "True" show_error_codes = "True" +attr_defined = "False" [tool.pytest.ini_options] testpaths = ["tests"] [tool.ruff] target-version = "py37" -line-length = 120 +line-length = 80 fix = true select = [ # flake8-2020 @@ -83,6 +85,8 @@ ignore = [ "E501", # DoNotAssignLambda "E731", + # Ternary + "SIM108", ] [tool.ruff.format]