Techdebt: allow test & lint to run for specific service

Makefile

@@ -1,17 +1,23 @@
 SHELL := /bin/bash
 
-SERVICE_NAME = "default"
-TEST_NAMES = "*"
+ifdef SERVICE_NAME
+TEST_SERVICE = and ${SERVICE_NAME}
+TEST_SERVICE_DIR = test_${SERVICE_NAME}
+TF_SERVICE_NAME = ${SERVICE_NAME}
+MYPY_EXPLICIT_FILES ?= moto/${SERVICE_NAME}/*.py*
+endif
+
+TF_SERVICE_NAME ?= default
+TEST_NAMES ?= "*"
 
+# Parallel tests will be run separate
+# -m tags are specified in tests/conftest.py, or on individual files/tests
 ifeq ($(TEST_SERVER_MODE), true)
-	# exclude test_kinesisvideoarchivedmedia
-	# because testing with moto_server is difficult with data-endpoint
-	TEST_EXCLUDE := --ignore tests/test_kinesisvideoarchivedmedia --ignore tests/test_acm --ignore tests/test_amp --ignore tests/test_awslambda --ignore tests/test_batch --ignore tests/test_ec2 --ignore tests/test_sqs
-	# Parallel tests will be run separate
-	PARALLEL_TESTS := ./tests/test_acm/ ./tests/test_acmpca/ ./tests/test_amp/ ./tests/test_awslambda ./tests/test_batch ./tests/test_ec2 ./tests/test_sqs
+	TEST_EXCLUDE := -m "not parallel and not parallel_server_only and not skip_server ${TEST_SERVICE}"
+	PARALLEL_TESTS := -m "parallel or parallel_server_only ${TEST_SERVICE}"
 else
-	TEST_EXCLUDE := --ignore tests/test_batch --ignore tests/test_ec2 --ignore tests/test_sqs
-	PARALLEL_TESTS := ./tests/test_batch ./tests/test_ec2 ./tests/test_sqs
+	TEST_EXCLUDE := -m "not parallel ${TEST_SERVICE}"
+	PARALLEL_TESTS := -m "parallel ${TEST_SERVICE}"
 endif
 
 init:
@@ -20,34 +26,36 @@ init:
 
 lint:
 	@echo "Running flake8..."
-	flake8 moto tests
+	flake8 moto/${SERVICE_NAME} tests/${TEST_SERVICE_DIR}
 	@echo "Running black... "
 	$(eval black_version := $(shell grep "^black==" requirements-dev.txt | sed "s/black==//"))
 	@echo "(Make sure you have black-$(black_version) installed, as other versions will produce different results)"
-	black --check moto/ tests/
+	black --check moto/${SERVICE_NAME} tests/${TEST_SERVICE_DIR}
 	@echo "Running pylint..."
-	pylint -j 0 moto tests
+	pylint -j 0 moto/${SERVICE_NAME} tests/${TEST_SERVICE_DIR}
 	@echo "Running MyPy..."
-	mypy --install-types --non-interactive
+	if [[ "${SKIP_MYPY}" == "" ]]; then mypy --install-types --non-interactive ${MYPY_EXPLICIT_FILES}; else echo "Skipping"; fi
 
 format:
 	black moto/ tests/
 
 test-only:
 	rm -f .coverage
 	rm -rf cover
-	pytest -sv --cov=moto --cov-report xml ./tests/ $(TEST_EXCLUDE)
+	pytest -sv --cov=moto --cov-report xml ./tests $(TEST_EXCLUDE) ${PYTEST_ARGS}
 	# https://github.com/aws/aws-xray-sdk-python/issues/196 - Run these tests separately without Coverage enabled
-	pytest -sv ./tests/test_xray
-	MOTO_CALL_RESET_API=false pytest --cov=moto --cov-report xml --cov-append -n 4 $(PARALLEL_TESTS)
+	if [[ "${SERVICE_NAME}" == "" || "${SERVICE_NAME}" == "xray" ]]; then pytest -sv ./tests/test_xray ${PYTEST_ARGS}; else echo "Skipping"; fi
+	MOTO_CALL_RESET_API=false pytest --cov=moto --cov-report xml --cov-append -n 4 $(PARALLEL_TESTS) ${PYTEST_ARGS}
 
 test: lint test-only
+	@echo "USAGE: make test [SERVICE_NAME=s3] [SKIP_MYPY=true]"
+	@echo "     defaults to all services and running mypy (not all services are mypy proof, see also setup.cfg)"
 
 terraformtests:
 	@echo "Make sure that the MotoServer is already running on port 4566 (moto_server -p 4566)"
 	@echo "USAGE: make terraformtests SERVICE_NAME=acm TEST_NAMES=TestAccACMCertificate"
 	@echo ""
-	cd tests/terraformtests && bin/run_go_test $(SERVICE_NAME) "$(TEST_NAMES)"
+	cd tests/terraformtests && bin/run_go_test $(TF_SERVICE_NAME) "$(TEST_NAMES)"
 
 test_server:
 	@TEST_SERVER_MODE=true pytest -sv --cov=moto --cov-report xml ./tests/

docs/docs/contributing/installation.rst

@@ -36,14 +36,11 @@ With all dependencies installed, run the following command to run all the tests
 
 Note that this may take awhile - there are many services, and each service will have a boatload of tests.
 
-To verify all tests pass for a specific service, for example for `s3`, run these commands manually:
+To verify all tests pass for a specific service, for example for `s3`, run the following command:
 
 .. code-block:: bash
 
-  flake8 moto/s3
-  black --check moto/s3 tests/test_s3
-  pylint tests/test_s3
-  pytest -sv tests/test_s3
+  make test SERVICE_NAME=s3
 
 If black fails, you can run the following command to automatically format the offending files:
 

setup.cfg

@@ -211,7 +211,130 @@ universal=1
 
 [tool:pytest]
 markers =
+    # special cases
     network: marks tests which require network connection
+    parallel: can be run in parallel
+    parallel_server_only: can be run in parallel (in server mode only)
+    skip_server: should not be run in server mode
+    # services
+    acm: acm service
+    acmpca: acmpca service
+    amp: amp service
+    apigateway: apigateway service
+    apigatewayv2: apigatewayv2 service
+    applicationautoscaling: applicationautoscaling service
+    appsync: appsync service
+    athena: athena service
+    autoscaling: autoscaling service
+    awslambda: awslambda service
+    batch: batch service
+    batch_simple: batch_simple service
+    budgets: budgets service
+    ce: ce service
+    cloudformation: cloudformation service
+    cloudfront: cloudfront service
+    cloudtrail: cloudtrail service
+    cloudwatch: cloudwatch service
+    codebuild: codebuild service
+    codecommit: codecommit service
+    codepipeline: codepipeline service
+    cognitoidentity: cognitoidentity service
+    cognitoidp: cognitoidp service
+    comprehend: comprehend service
+    config: config service
+    databrew: databrew service
+    datapipeline: datapipeline service
+    datasync: datasync service
+    dax: dax service
+    dms: dms service
+    ds: ds service
+    dynamodb: dynamodb service
+    dynamodb_v20111205: dynamodb_v20111205 service
+    dynamodbstreams: dynamodbstreams service
+    ebs: ebs service
+    ec2: ec2 service
+    ec2instanceconnect: ec2instanceconnect service
+    ecr: ecr service
+    ecs: ecs service
+    efs: efs service
+    eks: eks service
+    elasticache: elasticache service
+    elasticbeanstalk: elasticbeanstalk service
+    elastictranscoder: elastictranscoder service
+    elb: elb service
+    elbv2: elbv2 service
+    emr: emr service
+    emrcontainers: emrcontainers service
+    emrserverless: emrserverless service
+    es: es service
+    events: events service
+    firehose: firehose service
+    forecast: forecast service
+    glacier: glacier service
+    glue: glue service
+    greengrass: greengrass service
+    guardduty: guardduty service
+    iam: iam service
+    identitystore: identitystore service
+    instance_metadata: instance_metadata service
+    iot: iot service
+    iotdata: iotdata service
+    kinesis: kinesis service
+    kinesisvideo: kinesisvideo service
+    kinesisvideoarchivedmedia: kinesisvideoarchivedmedia service
+    kms: kms service
+    logs: logs service
+    managedblockchain: managedblockchain service
+    mediaconnect: mediaconnect service
+    medialive: medialive service
+    mediapackage: mediapackage service
+    mediastore: mediastore service
+    mediastoredata: mediastoredata service
+    meteringmarketplace: meteringmarketplace service
+    moto_api: moto_api service
+    moto_server: moto_server service
+    mq: mq service
+    neptune: neptune service
+    opsworks: opsworks service
+    organizations: organizations service
+    packages: packages service
+    personalize: personalize service
+    pinpoint: pinpoint service
+    polly: polly service
+    quicksight: quicksight service
+    ram: ram service
+    rds: rds service
+    redshift: redshift service
+    redshiftdata: redshiftdata service
+    rekognition: rekognition service
+    resourcegroups: resourcegroups service
+    resourcegroupstaggingapi: resourcegroupstaggingapi service
+    route53: route53 service
+    route53resolver: route53resolver service
+    s3: s3 service
+    s3bucket_path: s3bucket_path service
+    s3control: s3control service
+    sagemaker: sagemaker service
+    sdb: sdb service
+    secretsmanager: secretsmanager service
+    servicediscovery: servicediscovery service
+    servicequotas: servicequotas service
+    ses: ses service
+    signer: signer service
+    sns: sns service
+    sqs: sqs service
+    ssm: ssm service
+    ssoadmin: ssoadmin service
+    stepfunctions: stepfunctions service
+    sts: sts service
+    support: support service
+    swf: swf service
+    textract: textract service
+    timestreamwrite: timestreamwrite service
+    transcribe: transcribe service
+    utilities: utilities service
+    wafv2: wafv2 service
+    xray: xray service
 
 [coverage:run]
 relative_files = True
@@ -229,7 +352,14 @@ disable = W,C,R,E
 enable = anomalous-backslash-in-string, arguments-renamed, dangerous-default-value, deprecated-module, function-redefined, import-self, redefined-builtin, redefined-outer-name, reimported, pointless-statement, super-with-arguments, unused-argument, unused-import, unused-variable, useless-else-on-loop, wildcard-import
 
 [mypy]
-files= moto/a*,moto/b*,moto/c*,moto/d*,moto/e*,moto/f*,moto/moto_api,moto/neptune
+files= moto/a*,moto/b*,moto/c*,moto/d*,moto/e*,moto/moto_api,moto/neptune
+exclude = (?x)(
+    ^moto/[g-l]
+    | ^moto/m(?!oto_api)  # all m*, except moto_api
+    | ^moto/n(?!eptune)   # all n*, except neptune
+    | ^moto/[o-z]
+ )
+
 show_column_numbers=True
 show_error_codes = True
 disable_error_code=abstract

tests/conftest.py

@@ -0,0 +1,149 @@
+import pytest
+
+SERVICES = (
+    "acm",
+    "acmpca",
+    "amp",
+    "apigateway",
+    "apigatewayv2",
+    "applicationautoscaling",
+    "appsync",
+    "athena",
+    "autoscaling",
+    "awslambda",
+    "batch",
+    "batch_simple",
+    "budgets",
+    "ce",
+    "cloudformation",
+    "cloudfront",
+    "cloudtrail",
+    "cloudwatch",
+    "codebuild",
+    "codecommit",
+    "codepipeline",
+    "cognitoidentity",
+    "cognitoidp",
+    "comprehend",
+    "config",
+    "databrew",
+    "datapipeline",
+    "datasync",
+    "dax",
+    "dms",
+    "ds",
+    "dynamodb",
+    "dynamodb_v20111205",
+    "dynamodbstreams",
+    "ebs",
+    "ec2",
+    "ec2instanceconnect",
+    "ecr",
+    "ecs",
+    "efs",
+    "eks",
+    "elasticache",
+    "elasticbeanstalk",
+    "elastictranscoder",
+    "elb",
+    "elbv2",
+    "emr",
+    "emrcontainers",
+    "emrserverless",
+    "es",
+    "events",
+    "firehose",
+    "forecast",
+    "glacier",
+    "glue",
+    "greengrass",
+    "guardduty",
+    "iam",
+    "identitystore",
+    "instance_metadata",
+    "iot",
+    "iotdata",
+    "kinesis",
+    "kinesisvideo",
+    "kinesisvideoarchivedmedia",
+    "kms",
+    "logs",
+    "managedblockchain",
+    "mediaconnect",
+    "medialive",
+    "mediapackage",
+    "mediastore",
+    "mediastoredata",
+    "meteringmarketplace",
+    "moto_api",
+    "moto_server",
+    "mq",
+    "neptune",
+    "opsworks",
+    "organizations",
+    "packages",
+    "personalize",
+    "pinpoint",
+    "polly",
+    "quicksight",
+    "ram",
+    "rds",
+    "redshift",
+    "redshiftdata",
+    "rekognition",
+    "resourcegroups",
+    "resourcegroupstaggingapi",
+    "route53",
+    "route53resolver",
+    "s3",
+    "s3bucket_path",
+    "s3control",
+    "sagemaker",
+    "sdb",
+    "secretsmanager",
+    "servicediscovery",
+    "servicequotas",
+    "ses",
+    "signer",
+    "sns",
+    "sqs",
+    "ssm",
+    "ssoadmin",
+    "stepfunctions",
+    "sts",
+    "support",
+    "swf",
+    "textract",
+    "timestreamwrite",
+    "transcribe",
+    "utilities",
+    "wafv2",
+    "xray",
+)
+
+
+EXTRA_MARKERS_TEST_DIRS = {
+    "acm": (pytest.mark.parallel_server_only,),
+    "acmpca": (pytest.mark.parallel_server_only,),
+    "amp": (pytest.mark.parallel_server_only,),
+    "awslambda": (pytest.mark.parallel_server_only,),
+    "batch": (pytest.mark.parallel,),
+    "ec2": (pytest.mark.parallel,),
+    # exclude test_kinesisvideoarchivedmedia
+    # because testing with moto_server is difficult with data-endpoint
+    "kinesisvideoarchivedmedia": (pytest.mark.skip_server,),
+    "sqs": (pytest.mark.parallel,),
+}
+
+
+def pytest_collection_modifyitems(items):  # noqa: SC200
+    for item in items:
+        for service, markers in EXTRA_MARKERS_TEST_DIRS.items():
+            if f"tests/test_{service}/" in item.nodeid:  # noqa: SC200
+                for marker in markers:
+                    item.add_marker(marker)
+        for service in SERVICES:
+            if f"tests/test_{service}/" in item.nodeid:  # noqa: SC200
+                item.add_marker(getattr(pytest.mark, service))
+            if f"_{service}.py" in item.nodeid:
+                item.add_marker(getattr(pytest.mark, service))