From e1ee4c3164317944b687c3492a712aa4a3dce6a3 Mon Sep 17 00:00:00 2001 From: Andrew DiLosa Date: Thu, 16 Nov 2023 21:11:57 -0800 Subject: [PATCH] update twitter api python-twitter appears to not handle the new twitter v2 changes, example usage from twitter is to just call the api directly https://github.com/bear/python-twitter/issues/722 https://github.com/twitterdev/Twitter-API-v2-sample-code/blob/main/Manage-Tweets/create_tweet.py#L51 which didn't work, so switched to Tweepy which is working. Additionally the new rate limits don't like the bot bulk-posting a whole update at once, so adjusted to just exit if we hit a rate limit. The dynamo cache should make sure we don't miss a post or repost. Last, the API apparently somehow now doesn't support tweets longer than 280 chars at all, so making sure we stay under budget --- Pipfile | 3 ++- Pipfile.lock | 58 +++++++++++++++++++++++----------------------------- handler.py | 58 +++++++++++++++++++++++----------------------------- 3 files changed, 54 insertions(+), 65 deletions(-) diff --git a/Pipfile b/Pipfile index 2986d13..49c8842 100644 --- a/Pipfile +++ b/Pipfile @@ -4,10 +4,11 @@ verify_ssl = true name = "pypi" [packages] -python-twitter = "*" requests = "*" +requests_oauthlib = "*" "boto3" = "*" feedparser = "*" +tweepy = "*" [dev-packages] "boto3" = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 1d0a77e..b1d55e5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4a6e89717d0a773a3f862257e71a268b7d7e1709642c50d7aeb4ae416e239f42" + "sha256": "b1ac678553c8dde63401db8ef2be0468647954297f25dbc47d6d8c5e0e9507cd" }, "pipfile-spec": 6, "requires": { @@ -18,28 +18,28 @@ "default": { "boto3": { "hashes": [ - "sha256:6617ac176efb21485ebc3a058a3a97feb1300141421ae3d1809562c4cac1d5f9", - "sha256:f3024bba9ac980007ba7b5f28a9734d111fb5466e2426ac76c5edbd6dedd8db2" + "sha256:85123ba6ccef12f8230bcd85bf730d3c4218e08e3cc4baaa0b3eae094703e77d", + "sha256:d038b19cbe29d488133351ee6eb36ee11a0934df8bcbc0892bbeb2c544a327a4" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==1.29.2" + "version": "==1.29.3" }, "botocore": { "hashes": [ - "sha256:0e231524e9b72169fe0b8d9310f47072c245fb712778e0669f53f264f0e49536", - "sha256:a68a33193d8cd59e3b2142bff632e562afc02f9c4417e3dcc81a6e1b1f47148e" + "sha256:115adb7edf61ad7083fd582ac749b761fa707758bbca94d42e4e6e92940b5d38", + "sha256:be622915db1dbf1d6d5ed907633471f9ed8f5399dd3cf333f9dc2b955cd3e80d" ], "markers": "python_version >= '3.7'", - "version": "==1.32.2" + "version": "==1.32.3" }, "certifi": { "hashes": [ - "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", - "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" + "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1", + "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474" ], "markers": "python_version >= '3.6'", - "version": "==2023.7.22" + "version": "==2023.11.17" }, "charset-normalizer": { "hashes": [ @@ -146,13 +146,6 @@ "markers": "python_version >= '3.6'", "version": "==6.0.10" }, - "future": { - "hashes": [ - "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.18.3" - }, "idna": { "hashes": [ "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", @@ -185,14 +178,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, - "python-twitter": { - "hashes": [ - "sha256:45855742f1095aa0c8c57b2983eee3b6b7f527462b50a2fa8437a8b398544d90", - "sha256:4a420a6cb6ee9d0c8da457c8a8573f709c2ff2e1a7542e2d38807ebbfe8ebd1d" - ], - "index": "pypi", - "version": "==3.5" - }, "requests": { "hashes": [ "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", @@ -232,6 +217,15 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, + "tweepy": { + "hashes": [ + "sha256:1f9f1707d6972de6cff6c5fd90dfe6a449cd2e0d70bd40043ffab01e07a06c8c", + "sha256:db6d3844ccc0c6d27f339f12ba8acc89912a961da513c1ae50fa2be502a56afb" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==4.14.0" + }, "urllib3": { "hashes": [ "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07", @@ -308,20 +302,20 @@ }, "boto3": { "hashes": [ - "sha256:6617ac176efb21485ebc3a058a3a97feb1300141421ae3d1809562c4cac1d5f9", - "sha256:f3024bba9ac980007ba7b5f28a9734d111fb5466e2426ac76c5edbd6dedd8db2" + "sha256:85123ba6ccef12f8230bcd85bf730d3c4218e08e3cc4baaa0b3eae094703e77d", + "sha256:d038b19cbe29d488133351ee6eb36ee11a0934df8bcbc0892bbeb2c544a327a4" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==1.29.2" + "version": "==1.29.3" }, "botocore": { "hashes": [ - "sha256:0e231524e9b72169fe0b8d9310f47072c245fb712778e0669f53f264f0e49536", - "sha256:a68a33193d8cd59e3b2142bff632e562afc02f9c4417e3dcc81a6e1b1f47148e" + "sha256:115adb7edf61ad7083fd582ac749b761fa707758bbca94d42e4e6e92940b5d38", + "sha256:be622915db1dbf1d6d5ed907633471f9ed8f5399dd3cf333f9dc2b955cd3e80d" ], "markers": "python_version >= '3.7'", - "version": "==1.32.2" + "version": "==1.32.3" }, "cfn-lint": { "hashes": [ @@ -984,7 +978,7 @@ "sha256:fd946abf3c31fb50eee07451a6aedbfff912fcd13cf357363f5b4e834cc5e71a", "sha256:fe58ef6a764de7b4b36edfc8592641f56e69b7163bba9f9c8089838ee596bfb2" ], - "markers": "python_version < '3.8' and implementation_name == 'cpython'", + "markers": "python_version < '3.8'", "version": "==1.5.5" }, "typing-extensions": { diff --git a/handler.py b/handler.py index 539afc6..0217a8c 100644 --- a/handler.py +++ b/handler.py @@ -1,13 +1,14 @@ import json import logging import os -import time from html.parser import HTMLParser import boto3 import feedparser -import twitter +import requests +import tweepy from botocore.client import Config +from requests_oauthlib import OAuth1Session logger = logging.getLogger() logger.setLevel(logging.INFO) @@ -27,56 +28,49 @@ def get_data(self): return "".join(self.fed) +s = MLStripper() + + def strip_tags(html): - s = MLStripper() s.feed(html) return s.get_data() -api = twitter.Api( - **{ - k: v - for k, v in json.loads( - boto3.client("s3", config=Config(signature_version="s3v4")) - .get_object(Bucket=os.environ["bucket"], Key="secrets.json")["Body"] - .read() - .decode("utf-8") - ).items() - if k - in { - "consumer_key", - "consumer_secret", - "access_token_key", - "access_token_secret", - } - } +secret = json.loads( + boto3.client("s3", config=Config(signature_version="s3v4")) + .get_object(Bucket=os.environ["bucket"], Key="secrets.json")["Body"] + .read() + .decode("utf-8") ) -posts_table = boto3.resource("dynamodb", region_name="us-west-2").Table(os.environ["PostsTableName"]) - - -def within(t: time.struct_time, minutes: int) -> bool: - return abs(time.mktime(time.gmtime()) - time.mktime(t)) <= (minutes * 60) +posts_table = boto3.resource("dynamodb", region_name="us-west-2").Table( + os.environ["PostsTableName"] +) def already_posted(guid: str) -> bool: return "Item" in posts_table.get_item(Key={"guid": guid}) +client = tweepy.Client(**secret) + +separator = "\n\n" +ellipsis = "... " + def lambda_handler(event, context): for entry in feedparser.parse("http://aws.amazon.com/new/feed/").entries: logger.info(f"Checking {entry.guid} - {entry.title}") if not already_posted(entry.guid): logger.info(f"Posting {entry.guid} - {entry.title}") try: - api.PostUpdate( - (entry.title + "\n\n" + strip_tags(entry.description))[:249] - + "... " - + entry.link, - verify_status_length=False, - ) + char_budget = 280 - len(entry.title) - len(entry.link) - len(separator) - len(ellipsis) + tweet_text = entry.title + separator + strip_tags(entry.description)[:char_budget] + ellipsis + entry.link + client.create_tweet(text=tweet_text) posts_table.put_item( Item={"guid": entry.guid, "title": entry.title, "link": entry.link} ) + except tweepy.TooManyRequests: + logger.exception("Too many requests, exiting lambda for now") + return except Exception: - logger.exception(f"Failed to post tweet") + logger.exception("Failed to post tweet")