Skip to content

Commit edf0ad3

Browse files
author
徳住友稜
committed
first
0 parents  commit edf0ad3

File tree

5 files changed

+294
-0
lines changed

5 files changed

+294
-0
lines changed

.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
__pycache__
2+
env
3+
.mypy_cache
4+
.vscode

.python-version

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3.6.8

README.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Qiita notification job with AWS Lambda and Dynamo DB Stream.
2+
this repository includes two code, which apply to Lambda one by one.
3+
4+
## /qiita_iine_collect/check_new_iine_dev.py
5+
- collect all articles iine by Qiita API v2
6+
- update logs in Dynamo DB to stream differences, which is target of notification
7+
8+
## /qiita_notification/send_notification.py
9+
- get stream data of Dynamo DB
10+
- notify via LINE Notify
Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import os
2+
from math import ceil
3+
from typing import List, Dict, Any, Union, Tuple
4+
import json
5+
from urllib.request import Request
6+
from urllib import request, parse, error
7+
from http.client import HTTPResponse
8+
import boto3
9+
from botocore.exceptions import ClientError
10+
11+
12+
class Response():
13+
"""Http Response Object"""
14+
15+
def __init__(self, res: HTTPResponse):
16+
self.body = self._json(res)
17+
self.status_code = self._status_code(res)
18+
self.headers = self._headers(res)
19+
20+
def _json(self, res: HTTPResponse):
21+
return json.loads(res.read())
22+
23+
def _status_code(self, res: HTTPResponse) -> int:
24+
return res.status
25+
26+
def _headers(self, res: HTTPResponse) -> Dict[str, str]:
27+
return dict(res.getheaders())
28+
29+
30+
def req_get(url, headers=None, params=None) -> Response:
31+
"""get request. simplified request function of Requests
32+
:return: Response object
33+
"""
34+
if params:
35+
url = '{}?{}'.format(url, parse.urlencode(params))
36+
37+
req = Request(url, headers=headers, method='GET')
38+
39+
with request.urlopen(req) as res:
40+
response = Response(res)
41+
return response
42+
43+
44+
def serialize_response(response: Response) -> List[Dict[str, Any]]:
45+
"""serialize response of Qiita API v2
46+
:param response:
47+
:return:
48+
"""
49+
keys = ['id', 'title', 'likes_count']
50+
return [
51+
{f: resp.get(f) for f in keys} for resp in response.body
52+
]
53+
54+
55+
def get_item(url: str, headers: Dict[str, str], **param) -> List[Dict[str, Any]]:
56+
"""get a item by Qiita API v2 and return the list of serialized response (dictionary)"""
57+
response = req_get(url, headers=headers, params=param)
58+
return serialize_response(response)
59+
60+
61+
def get_items(token: str, per_page=1, url='https://qiita.com/api/v2/authenticated_user/items') -> List[Dict[str, Any]]:
62+
"""ページネーションして認証ユーザの全ての記事を取得する
63+
:return: 記事のリスト
64+
"""
65+
headers = {'Authorization': 'Bearer {}'.format(token)}
66+
67+
response: Response = req_get(url, headers=headers, params={'page': 1, 'per_page': per_page})
68+
items = serialize_response(response)
69+
tot_count = int(response.headers['Total-Count'])
70+
tot_pages = ceil(tot_count / per_page)
71+
if tot_pages <= 1:
72+
return items
73+
74+
for page in range(2, tot_pages + 1):
75+
items += get_item(url, headers, page=page, per_page=per_page)
76+
return items
77+
78+
79+
def update_logs(items: List[Dict[str, Any]]):
80+
"""Update the number of iine in Dynamo DB
81+
If item ID do not exist in Dynamo DB, insert them in it
82+
"""
83+
dynamodb = boto3.resource('dynamodb')
84+
85+
table = dynamodb.Table('iine_qiita_logs')
86+
87+
for item in items:
88+
ids = item.get('id')
89+
title = item.get('title')
90+
iine = item.get('likes_count')
91+
92+
try:
93+
response = table.update_item(
94+
Key={
95+
'ids': ids
96+
},
97+
UpdateExpression="set iine = :newiine, title = :title",
98+
ConditionExpression="attribute_not_exists(ids) or iine <> :newiine",
99+
ExpressionAttributeValues={
100+
":newiine": iine,
101+
":title": title
102+
},
103+
)
104+
except ClientError as e:
105+
if e.response['Error']['Code'] == "ConditionalCheckFailedException":
106+
print(e.response['Error']['Message'])
107+
else:
108+
raise
109+
110+
111+
def main(client, content):
112+
"""this is handler function for Lambda"""
113+
qiita_token: str = os.environ['QIITA_TOKEN']
114+
url: str = os.environ['QIITA_URL']
115+
per_page = int(os.environ['PER_PAGE'])
116+
117+
items: List[Dict[str, Any]] = get_items(qiita_token, per_page=per_page, url=url)
118+
update_logs(items)
119+
return {
120+
'statusCode': 200
121+
}
Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
import json
2+
import os
3+
from math import ceil
4+
from typing import List, Dict, Any, Union, Tuple
5+
import json
6+
from urllib.request import Request
7+
from urllib import request, parse, error
8+
from http.client import HTTPResponse
9+
10+
11+
class Response():
12+
"""Http Response Object"""
13+
14+
def __init__(self, res: HTTPResponse):
15+
self.body = self._json(res)
16+
self.status_code = self._status_code(res)
17+
self.headers = self._headers(res)
18+
19+
def _json(self, res: HTTPResponse):
20+
return json.loads(res.read())
21+
22+
def _status_code(self, res: HTTPResponse) -> int:
23+
return res.status
24+
25+
def _headers(self, res: HTTPResponse) -> Dict[str, str]:
26+
return dict(res.getheaders())
27+
28+
29+
def req_get(url: str, headers=None, params=None) -> Response:
30+
"""get request. simplified request function of Requests
31+
:return: Response object
32+
"""
33+
if params:
34+
url = '{}?{}'.format(url, parse.urlencode(params))
35+
36+
req = Request(url, headers=headers, method='GET')
37+
38+
with request.urlopen(req) as res:
39+
response = Response(res)
40+
return response
41+
42+
43+
def req_post(url: str, data: Dict[str, Any], headers=None, params=None) -> Response:
44+
"""post request. simplified request function of Requests
45+
:return: Response object
46+
"""
47+
if headers.get('Content-Type') == 'application/x-www-form-urlencoded':
48+
encoded_data = parse.urlencode(data).encode()
49+
50+
else:
51+
encoded_data = json.dumps(data).encode()
52+
53+
req = Request(url, data=encoded_data, headers=headers, method='POST')
54+
55+
with request.urlopen(req) as res:
56+
response = Response(res)
57+
return response
58+
59+
60+
def serialize_record(record: Dict[str, Any]) -> Dict[str, Any]:
61+
"""serialize data of Dynamo DB Stream
62+
:return:
63+
"""
64+
if record.get('eventName') != 'MODIFY':
65+
return {}
66+
67+
past = record.get('dynamodb', {}).get('OldImage')
68+
past_iine = int(past.get('iine', {}).get('N', 0))
69+
ids = past.get('ids', {}).get('S', '')
70+
71+
new = record.get('dynamodb', {}).get('NewImage')
72+
title = new.get('title', {}).get('S', '')
73+
return {
74+
'ids': ids,
75+
'title': title,
76+
'past_iine': past_iine
77+
}
78+
79+
80+
def serialize_response_name(response: Response, num: int, title: str) -> Dict[str, Any]:
81+
"""serialize iine data of Qiita API v2
82+
:param response:
83+
:return:
84+
"""
85+
size = len(response.body) - num
86+
if size <= 0:
87+
users: List[str] = []
88+
89+
new_iine = response.body[:size]
90+
users = [
91+
resp.get('user', {}).get('id') for resp in new_iine
92+
]
93+
return {
94+
'title': title,
95+
'users': users
96+
}
97+
98+
99+
def get_new_iine(item: Dict[str, Any], token: str) -> Dict[str, Any]:
100+
"""HTTP request to Qiita API v2
101+
:params:
102+
:return:
103+
"""
104+
headers = {'Authorization': 'Bearer {}'.format(token)}
105+
ids = item.get('ids', '')
106+
past_iine = item.get('past_iine', 0)
107+
url = f'https://qiita.com/api/v2/items/{ids}/likes'
108+
109+
response = req_get(url, headers=headers)
110+
title: str = item.get('title', '')
111+
resp = serialize_response_name(response, past_iine, title)
112+
return resp
113+
114+
115+
def deserialize_response_name(response: Dict[str, Any], max_length=20) -> str:
116+
"""deserialize text for LINE Notify
117+
:param max_length: max sentence length
118+
:return:
119+
"""
120+
names = ", ".join(response.get('users', []))
121+
title = response.get('title', '')
122+
title = f"{title}" if len(title) <= max_length else f"{title[:max_length]}..."
123+
return f"\n{names}が「{title}」にいいねしました。"
124+
125+
126+
def send_notification(message: str, token: str):
127+
"""send notification by LINE notify"""
128+
url = 'https://notify-api.line.me/api/notify'
129+
130+
headers = {
131+
'Authorization': 'Bearer {}'.format(token),
132+
'Content-Type': 'application/x-www-form-urlencoded'
133+
}
134+
msg = {
135+
'message': message
136+
}
137+
response = req_post(url, data=msg, headers=headers)
138+
return response.body
139+
140+
141+
def lambda_handler(event, context):
142+
"""main handler for Lambda"""
143+
qiita_token = os.environ["QIITA_TOKEN"]
144+
line_token = os.environ["LINE_TOKEN"]
145+
146+
records = event.get('Records', [])
147+
for record in records:
148+
serialized_data = serialize_record(record)
149+
if not serialized_data:
150+
continue
151+
new_iines = get_new_iine(serialized_data, qiita_token)
152+
if len(new_iines.get('users')) == 0:
153+
continue
154+
send_notification(deserialize_response_name(new_iines), line_token)
155+
156+
return {
157+
'statusCode': 200,
158+
}

0 commit comments

Comments
 (0)