Kategori: blandet

Categories
blandet

Ting du ikke vil vide om kortspillet Krig

Hvis man tilfældigvis har et barn i 5-årsalderen, kan man spille kortspillet Krig. Kortene blandes og deles ligeligt mellem 2 spillere, hver spiller vender et kort fra sin bunke samtidig, højeste kort vender, hvis kortene er lige høje, er der krig. Det er så enkelt, at man lige så godt kunne få en computer til at spille det.

Derfor skrev jeg et lille program i Python, der kan simulere kortspillet.

Jeg opdagede et hul i reglerne: Der er ingen steder, der beskriver, hvad der sker, når en spiller ikke har kort nok til at deltage i en krig (eller en dobbelt-krig, tredobbelt-krig, osv.) Jeg besluttede, at hvis en spiller på et tidspunkt mangler kort til at kunne deltage, taber den spiller, der ikke har kort nok til at deltage. I den meget sjældne situation, at begge spillere ikke har nok kort til at deltage (en mange-mange-dobbelt-krig i starten af spillet), vinder den spiller, der har flest kort. Har begge spillere lige mange kort, bliver det uafgjort.

Jeg fik computeren til at spille 1 million spil Krig, og her er hvad jeg kan fortælle dig om Krig, som du ikke vil vide:

  • Det gennemsnitlige antal dueller i et spil krig er 177
  • Spillet med flest dueller havde 1.825 dueller
  • Spillet med færrest havde 4 dueller
  • Spilleren med den højeste sum af kort efter kortene blev blandet vandt 573.405 gange
  • Spilleren med den laveste sum af kort vandt 397.602 gange
  • I løbet af spillene blev der spillet:
    • Enkeltkrig: 12.366.762 gange
    • Dobbeltkrig: 888.024 gange
    • Trippelkrig: 60.727 gange
    • Firdobbeltkrig: 3.852 gange
    • Femdobbeltkrig: 206 gange
    • Seksdobbeltkrig: 10 gange

Her er programmet:

import random

krig1 = 0
krig2 = 0
krig3 = 0
krig4 = 0
krig5 = 0
krig6 = 0
krig7 = 0

number_of_plays_list = []
not_war = 0
war = 0

highest_deck_won = 0
highest_deck_lost = 0
equal_games = 0

i = 0
number_of_games = 1000000

while i <= number_of_games:
	number_of_plays_counter = 0
	deck = [2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11,12,12,12,12,13,13,13,13,14,14,14,14]
	random.shuffle(deck)

	player_a_deck = deck[0:26]
	player_b_deck = deck[26:52]

	if sum(player_a_deck) > sum(player_b_deck):
		highest_deck = "a"
	elif sum(player_a_deck) < sum(player_b_deck):
		highest_deck = "b"
	else:
		highest_deck = "equal"

	while len(player_a_deck) > 0 and len(player_b_deck) > 0:
		number_of_plays_counter += 1
		if player_a_deck[0] > player_b_deck[0]:
			not_war += 1
			player_a_deck.append(player_a_deck[0])
			player_a_deck.append(player_b_deck[0])
			del player_a_deck[0]
			del player_b_deck[0]
		elif player_a_deck[0] < player_b_deck[0]:
			not_war += 1
			player_b_deck.append(player_b_deck[0])
			player_b_deck.append(player_a_deck[0])
			del player_a_deck[0]
			del player_b_deck[0]
		elif player_a_deck[0] == player_b_deck[0]:
			war += 1
			krig1 += 1
			if len(player_a_deck) >= 5 and len(player_b_deck) >= 5:
				if player_a_deck[4] > player_b_deck[4]:
					player_a_deck.extend(player_a_deck[0:5])
					player_a_deck.extend(player_b_deck[0:5])
					del player_a_deck[0:5]
					del player_b_deck[0:5]
				elif player_a_deck[4] < player_b_deck[4]:
					player_b_deck.extend(player_b_deck[0:5])
					player_b_deck.extend(player_a_deck[0:5])
					del player_a_deck[0:5]
					del player_b_deck[0:5]
				elif player_a_deck[4] == player_b_deck[4]:
					krig2 += 1
					if len(player_a_deck) >= 9 and len(player_b_deck) >= 9:			
						if player_a_deck[8] > player_b_deck[8]:
							player_a_deck.extend(player_a_deck[0:9])
							player_a_deck.extend(player_b_deck[0:9])
							del player_a_deck[0:9]
							del player_b_deck[0:9]
						elif player_a_deck[8] < player_b_deck[8]:
							player_b_deck.extend(player_b_deck[0:9])
							player_b_deck.extend(player_a_deck[0:9])
							del player_a_deck[0:9]
							del player_b_deck[0:9]	
						elif player_a_deck[8] == player_b_deck[8]:
							krig3 += 1
							if len(player_a_deck) >= 13 and len(player_b_deck) >= 13:
								if player_a_deck[12] > player_b_deck[12]:
									player_a_deck.extend(player_a_deck[0:13])
									player_a_deck.extend(player_b_deck[0:13])
									del player_a_deck[0:13]
									del player_b_deck[0:13]
								elif player_a_deck[12] < player_b_deck[12]:
									player_b_deck.extend(player_b_deck[0:13])
									player_b_deck.extend(player_a_deck[0:13])
									del player_a_deck[0:13]
									del player_b_deck[0:13]	
								elif player_a_deck[12] == player_b_deck[12]:
									krig4 += 1
									if len(player_a_deck) >= 17 and len(player_b_deck) >= 17:
										if player_a_deck[16] > player_b_deck[16]:
											player_a_deck.extend(player_a_deck[0:17])
											player_a_deck.extend(player_b_deck[0:17])
											del player_a_deck[0:17]
											del player_b_deck[0:17]
										elif player_a_deck[16] < player_b_deck[16]:
											player_b_deck.extend(player_b_deck[0:17])
											player_b_deck.extend(player_a_deck[0:17])
											del player_a_deck[0:17]
											del player_b_deck[0:17]
										elif player_a_deck[16] == player_b_deck[16]:
											krig5 += 1
											if len(player_a_deck) >= 21 and len(player_b_deck) >= 21:
												if player_a_deck[20] > player_b_deck[20]:
													player_a_deck.extend(player_a_deck[0:21])
													player_a_deck.extend(player_b_deck[0:21])
													del player_a_deck[0:21]
													del player_b_deck[0:21]
												elif player_a_deck[20] < player_b_deck[20]:
													player_b_deck.extend(player_b_deck[0:21])
													player_b_deck.extend(player_a_deck[0:21])
													del player_a_deck[0:21]
													del player_b_deck[0:21]										
												elif player_a_deck[20] == player_b_deck[20]:
													krig6 += 1
													if len(player_a_deck) >= 25 and len(player_b_deck) >= 25:
														if player_a_deck[24] > player_b_deck[24]:
															player_a_deck.extend(player_a_deck[0:25])
															player_a_deck.extend(player_b_deck[0:25])
															del player_a_deck[0:25]
															del player_b_deck[0:25]
														elif player_a_deck[24] < player_b_deck[24]:
															player_b_deck.extend(player_b_deck[0:25])
															player_b_deck.extend(player_a_deck[0:25])
															del player_a_deck[0:25]
															del player_b_deck[0:25]
														elif player_a_deck[24] == player_b_deck[24]:
															krig7 += 1
															break
													else:
														break
											else:
												break
									else:
										break
							else:
								break
					else:
						break
			else:
				break
	if len(player_a_deck) > len(player_b_deck):
		if highest_deck == "a":
			highest_deck_won += 1
		elif highest_deck == "b"	:
			highest_deck_lost += 1
	elif len(player_a_deck) < len(player_b_deck):
		if highest_deck == "a":
			highest_deck_lost += 1
		elif highest_deck == "b"	:
			highest_deck_won += 1
	else:
		equal_games += 1
	number_of_plays_list.append(number_of_plays_counter)
	i += 1
	print(i/number_of_games)

print("Der blev spillet {} spil".format(number_of_games))
print("Det gennemsnitlige antal dueller var {}".format(sum(number_of_plays_list)/len(number_of_plays_list)))
print("Det højeste antal dueller var {}".format(max(number_of_plays_list)))
print("Det laveste antal dueller var {}".format(min(number_of_plays_list)))
print("Den spiller med højest sum af kort vandt {} gange".format(highest_deck_won))
print("Den spiller med højest sum af kort tabte {} gange".format(highest_deck_lost))
print(krig1, krig2, krig3, krig4, krig5, krig6, krig7)
print(not_war, war)
print("Uafgjorte spil: {}".format(equal_games))
Categories
blandet

Hent valutadata fra Nordnet med Python

Nordnet har fået en ny hjemmeside med en markedsoversigt, hvor man blandt andet kan finde valutakurser. Dem ville jeg gerne have fat i til et Excelark 🙂

Jeg trykkede F12 i min browser for at undersøge, hvad der sker, når jeg klikker “Valutaer” på siden, og hvad der krævedes af cookies og klient-identifikation for at få data tilbage. (Du kan læse mere om metoden i mange af mine andre programmeringsindlæg.)

Det endte med dette program, der genererer en CSV-fil med den seneste valutakurs for en række almindelige valuter fra Nordnet:

# -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com
""" This program gets currency data from Nordnet.
Handy for exporting to Excel with as few manual steps as possible """
import requests 

# Creates a dictionary to use for cookies	
cookies = {}

# Sets NEXT cookie
url = 'https://www.nordnet.dk/markedet'
r = requests.get(url)
cookies['NEXT'] = r.cookies['NEXT']

# Requests currency data
headers = {'client-id': 'NEXT'}

# Gets currency data
url = 'https://www.nordnet.dk/api/2/instrument_search/query/indicator?entity_type=CURRENCY&amp;apply_filters=market_overview_group%3DDK_GLOBAL_MO'
r = requests.get(url, cookies=cookies, headers=headers)
currencies = r.json()

# Generate CSV output of last value by looping through currencies
output = "navn;senest\n"
for currency in currencies['results']:
	name = currency['instrument_info']['name']
	price = str(currency['price_info']['last']['price'])	
	price = price.replace(".",",")
	output += name + ";" + price + "\n"

# Write CSV output to file #
with open("currency.csv", "w", encoding='utf8') as fout:
	fout.write(output)
Categories
blandet

Hent data om dit elektricitetsforbrug fra Ørsted med Python

Dette lille Python-program, genererer CSV-filer med dit elektricitetsforbrugsdata fra Ørsted (tidligere DONG), hvis du har en fjernaflæst måler. Du kan fx bruge programmet, hvis du har lyst til at holde øje med dit forbrug i et Excel-dokument.

Det færdige program

# -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com
""" This program gets your Ørsted electricity consumption data and saves it to CSV"""

import requests 
from datetime import datetime
from datetime import date
from datetime import timedelta

# USER ACCOUNT AND PERIOD DATA. SHOULD BE EDITED FOR YOUR NEEDS #

# User account credentials
user = ''	#E-mail address
password = ''		#Password

# Start date and date today used for consumption data
startdate = '2019-01-01'
today = date.today()
enddate = datetime.strftime(today, '%Y-%m-%d')

# API LOGIN #
url = 'https://api.obviux.dk/v2/authenticate'

headers = {
	'X-Customer-Ip': '0.0.0.0'
	}
	
credentials = {
	'customer': user,
	'password': password
	}

request = requests.post(url, headers=headers, json=credentials)
response = request.json()

# Save data for further API requests
external_id = response['external_id']
headers['Authorization'] = response['token']

# GET EAN #
url = 'https://api.obviux.dk/v2/deliveries'
request = requests.get(url, headers=headers)
response = request.json()

# Assuming only one Ørsted agreement, save ean value for further API queries for that agreement
# In case of more than one agreement, loop through list and save values instead
ean = response[0]['ean']

# API CALL #
base_url = 'https://capi.obviux.dk/v1/consumption/customer/'
id_ean = external_id + '/ean/' + ean + '/'

# There's limits on periods for each type of consumption data, so we create lists of periods
startdate_datetime = datetime.strptime(startdate, '%Y-%m-%d')
enddate_datetime = datetime.strptime(enddate, '%Y-%m-%d')

# The number of days for each period
days_for_type = {
	"hourly": 15,
	"daily": 370,
	"weekly": 420,
	"monthly": 1860,
	"yearly": 1830	
	}

def get_consumption_data(type):
	# Loop that returns a list of periods (dates) to request
	periods = []
	loop = True	
	start_of_periods = startdate_datetime
	days = days_for_type[type]

	while loop == True:
		start = datetime.strftime(start_of_periods, '%Y-%m-%d')
		end = start_of_periods + timedelta(days=days)
		# Loop ends and replaces end value if the calculated end date is later than what the user is looking for
		if end > enddate_datetime:
			end = enddate
			loop = False
		# Every weekly period ends on a Sunday and next starts on Monday
		elif type == "weekly" and not end.weekday() == 6:
			correction = end.weekday() + 1
			end -= timedelta(days=correction)
			end = datetime.strftime(end, '%Y-%m-%d')
			start_of_periods += timedelta(days=days + 1 - correction)
		# All months end on last day of month and next period starts the 1st of next month
		elif type == "monthly":
			day_in_month = end.day
			end -= timedelta(days=day_in_month)
			start_of_periods = end + timedelta(days=1)
			end = datetime.strftime(end, '%Y-%m-%d')
		# All yearly periods end on 31st December and next period starts next year January 1st
		elif type == "yearly":
			start_of_periods = datetime.strptime(str(end.year)+"-01-01", '%Y-%m-%d')
			end = str(end.year-1) + "-12-31"
		# Covers hourly and daily periods and weeks ending on Sunday
		else:
			end = datetime.strftime(end, '%Y-%m-%d')
			start_of_periods += timedelta(days=days + 1)
		periods.append([start, end])

	# API requests to cover requested periods
	url = base_url + id_ean + type
	responses = []
	for period in periods:
		params = {
			'from': period[0],
			'to': period[1]
			}
		request = requests.get(url, headers=headers, params=params)
		response = request.json()
		responses.append(response)

	# Write responses to CSV
	if type == "hourly":
		output = "date;hour;amount\n"
	elif type == "weekly":
		output = "date;weeknum;amount\n"
	else:
		output = "date;amount\n"
	for entry in responses:
		for data in entry['data']:
			if data['consumptions']:
				for d in data['consumptions']:
					start = datetime.strptime(d['start'], '%Y-%m-%dT%H:%M:%S.%f%z')
					start_in_timezone = datetime.astimezone(start).strftime("%Y-%m-%d %H:%M")
					amount = str(d['kWh']).replace(".",",")
					if type == "hourly":
						hour = datetime.astimezone(start).strftime("%H")
						output += start_in_timezone + ";" + hour + ";" + amount + "\n"
					elif type == "weekly":
						weeknum = datetime.astimezone(start).strftime("%V")
						output += start_in_timezone + ";" + weeknum + ";" + amount + "\n"
					else:
						output += start_in_timezone + ";" + amount + "\n"
		# Special case for year when change is done from manual to automatic consumption reading
		if type == "yearly":
			if entry['readings']:
				for ent in entry['readings']:
					if ent['readings']:
						for e in ent['readings']:
							if e['consumption']:
								date = e['startdate'] + " 00:00"
								amount = str(e['consumption']).replace(".",",")
								output += date + ";" + amount + "\n"
	filename = type + ".csv"
	with open(filename, "w", encoding='utf8') as fout:
		fout.write(output)

# Loop to cycle through all consumption endpoints
endpoints = ["hourly", "daily", "weekly", "monthly", "yearly"]
for endpoint in endpoints:
	get_consumption_data(endpoint)
Categories
blandet

Wallnots Twitter-bot: Version 2

Det er ikke mange dage siden, at Wallnot.dk‘s Twitter-bot gik i luften. Du kan finde botten her og mit indlæg om den her.

Robotten virkede sådan set fint nok, men pga. en begrænsning i Twitter’s API på 250 forespørgsler per måned, kunne jeg kun opdatere 4 gange i døgnet, og det er jo ret sjældent (det gamle program lavede 2 forespørgsler, hver gang det blev kørt, dvs. 30 dage * 4 opdateringer * 2 forespørgsler = 240 forespørgsler).

Heldigvis fandt jeg TWINT, et Python-modul der gør det nemt at hente data fra Twitter uden at gøre brug af Twitter’s API med dets kedelige begrænsninger.

Med genbrug af det meste af min gamle kode, har jeg nu lavet en version af robotten, der kan køre lige så tit, jeg har lyst til. Jeg har foreløbig sat den til at køre 4 gange i timen.

For sjov skyld har jeg også tilføjet en række venlige adjektiver om abonnenterne på Politiken og Zetland, som programmet vælger tilfældigt mellem, hver gang det lægger et link på Twitter.

Den færdige kode

Her er den færdige kode, hvis du er interesseret.

# -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com

import requests
from bs4 import BeautifulSoup
from datetime import datetime
from datetime import date
from datetime import timedelta
import json
import time
import random
import twint	# https://github.com/twintproject/twint
from TwitterAPI import TwitterAPI

# CONFIGURATION #
# List to store articles to post to Twitter
articlestopost = []

# Yesterday's date variable
yesterday = date.today() - timedelta(days=1)
since = yesterday.strftime("%Y-%m-%d")

# Twint configuration
c = twint.Config()
c.Hide_output = True
c.Store_object = True
c.Since = since

# API LOGIN
client_key = ''
client_secret = ''
access_token = ''
access_secret = ''
api = TwitterAPI(client_key, client_secret, access_token, access_secret)


# POLITIKEN #
# Run search
searchterm = "politiken.dk/del"
c.Search = searchterm
twint.run.Search(c)
tweets = twint.output.tweets_object

# Add urls in tweets to list and remove any duplicates from list
urllist = []
for tweet in tweets:
	for url in tweet.urls:
		if searchterm in url:
			urllist.append(url)

urllist = list(set(urllist))

# Only proces urls that were not in our last Twitter query
proceslist = []
with open("./pol_lastbatch.json", "r", encoding="utf8") as fin:
	lastbatch = list(json.load(fin))
	for url in urllist:
		if url not in lastbatch:
			proceslist.append(url)
# Save current query to use for next time
with open("./pol_lastbatch.json", "wt", encoding="utf8") as fout:
	lastbatch = json.dumps(urllist)
	fout.write(lastbatch)

# Request articles and get titles and dates and sort by dates
articlelist = []
titlecheck = []

for url in proceslist:
	try:
		data = requests.get(url)
		result = data.text
		if '"isAccessibleForFree": "True"' not in result:
			soup = BeautifulSoup(result, "lxml")
			# Finds titles and timestamps
			title = soup.find('meta', attrs={'property':'og:title'})
			title = title['content']
			timestamp = soup.find('meta', attrs={'property':'article:published_time'})
			timestamp = timestamp['content']
			dateofarticle = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S%z')
			realurl = data.history[0].headers['Location']
			if title not in titlecheck:
				articlelist.append({"title": title, "url": realurl, "date": dateofarticle})
				titlecheck.append(title)			
	except Exception as e:
		print(url)
		print(e)
			
articlelist_sorted = sorted(articlelist, key=lambda k: k['date']) 

# Check if article is already posted and update list of posted articles
with open("./pol_published.json", "r", encoding="utf8") as fin:
	alreadypublished = list(json.load(fin))
	# File below used for paywall.py to update wallnot.dk
	with open("./pol_full_share_links.json", "r", encoding="utf8") as finalready:	
		alreadypublishedalready = list(json.load(finalready))
		for art in articlelist_sorted:
			url = art['url']
			token = url.index("?shareToken")
			url = url[:token]
			if url not in alreadypublished:
				alreadypublished.append(url)
				articlestopost.append(art)
				alreadypublishedalready.append(art['url'])
		# Save updated already published links
		with open("./pol_published.json", "wt", encoding="utf8") as fout:
			alreadypublishedjson = json.dumps(alreadypublished)
			fout.write(alreadypublishedjson)
		with open("./pol_full_share_links.json", "wt", encoding="utf8") as fout:
			alreadypublishedjson = json.dumps(alreadypublishedalready)
			fout.write(alreadypublishedjson)


# ZETLAND #
# Run search
searchterm = "zetland.dk/historie"
c.Search = searchterm
twint.run.Search(c)
tweets = twint.output.tweets_object

# Add urls in tweets to list and remove any duplicates from list
urllist = []
for tweet in tweets:
	for url in tweet.urls:
		if searchterm in url:
			urllist.append(url)

urllist = list(set(urllist))

# Only proces urls that were not in our last Twitter query
proceslist = []
with open("./zet_lastbatch.json", "r", encoding="utf8") as fin:
	lastbatch = list(json.load(fin))
	for url in urllist:
		if url not in lastbatch:
			proceslist.append(url)
# Save current query to use for next time
with open("./zet_lastbatch.json", "wt", encoding="utf8") as fout:
	lastbatch = json.dumps(urllist)
	fout.write(lastbatch)

# Request articles and get titles and dates and sort by dates
articlelist = []
titlecheck = []

for url in proceslist:
	try:
		data = requests.get(url)
		result = data.text
		soup = BeautifulSoup(result, "lxml")
		title = soup.find('meta', attrs={'property':'og:title'})
		title = title['content']
		timestamp = soup.find('meta', attrs={'property':'article:published_time'})
		timestamp = timestamp['content']
		timestamp = timestamp[:timestamp.find("+")]
		dateofarticle = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f')
		if title not in titlecheck:
			articlelist.append({"title": title, "url": url, "date": dateofarticle})
			titlecheck.append(title)
	except Exception as e:
		print(url)
		print(e)
			
articlelist_sorted = sorted(articlelist, key=lambda k: k['date']) 

# Check if article is already posted and update list of posted articles
with open("./zet_published.json", "r", encoding="utf8") as fin:
	alreadypublished = list(json.load(fin))
	for art in articlelist_sorted:
		title = art['title']
		if title not in alreadypublished:
			alreadypublished.append(title)
			articlestopost.append(art)
	# Save updated already published links
	with open("./zet_published.json", "wt", encoding="utf8") as fout:
		alreadypublishedjson = json.dumps(alreadypublished, ensure_ascii=False)
		fout.write(alreadypublishedjson)


# POST TO TWITTER #
friendlyterms = ["flink","rar","gavmild","velinformeret","intelligent","sød","afholdt","bedårende","betagende","folkekær","godhjertet","henrivende","smagfuld","tækkelig","hjertensgod","graciøs","galant","tiltalende","prægtig","kær","godartet","human","indtagende","fortryllende","nydelig","venlig","udsøgt","klog","kompetent","dygtig","ejegod","afholdt","omsorgsfuld","elskværdig","prægtig","skattet","feteret"]
enjoyterms = ["God fornøjelse!", "Nyd den!", "Enjoy!", "God læsning!", "Interessant!", "Spændende!", "Vidunderligt!", "Fantastisk!", "Velsignet!", "Glæd dig!", "Læs den!", "Godt arbejde!", "Wauv!"]

if articlestopost:
	for art in articlestopost:
		if "zetland" in art['url']:
			medium = "Zetland"
		else:
			medium = "Politiken"
		friendlyterm = random.choice(friendlyterms)
		enjoyterm = random.choice(enjoyterms)
		status = "En " + friendlyterm + " abonnent på " + medium + " har delt en artikel. " + enjoyterm + " " + art['url']
		r = api.request('statuses/update', {'status': status})
		time.sleep(15)
Categories
blandet

Wallnots nye Twitter-robot

Opdatering: Jeg har lavet en ny, forbedret udgave af robottten. Læs om den her.

Både Zetland og Politiken har en feature, hvor abonnenter kan dele betalingsartikler med venner, bekendte og offentligheden. Artiklen får en unik URL, som låser op for betalingsmuren og lader alle og enhver læse artiklen.

Jeg tænkte at Wallnot – min hjemmeside med artikler, der ikke er bag betalingsmur – trængte til at være mere til stede på sociale medier.

Derfor har jeg lavet en robot, der gennemsøger Twitter for delte artikler fra Zetland og Politiken – og deler links’ne som tweets. Robotten opdater ca. på klokkeslettene 8.25, 12.25, 16.25 og 20.25. Det ville være skønt at kunne opdatere flere gange i døgnet, men så vil Twitter have penge.

Du finder Wallnots nye Twitter-robot her: https://twitter.com/wallnot_dk

Sådan ser det ud, når Wallnots robot tweeter.

Jeg brugte Python og modulet TwitterAPI. Hvis du selv vil køre programmet, skal du lave en udviklerkonto og en app hos Twitter. Se
https://developer.twitter.com/.

Her er det færdige program. 

# -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com
# THIS PROGRAM POSTS NEW SHARED ARTICLES FROM ZETLAND.DK AND POLITIKEN.DK TO TWITTER

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import json
import time
from nested_lookup import nested_lookup
from TwitterAPI import TwitterAPI

articlestopost = []

# API LOGIN - INSERT YOUR OWN VALUES HERE
client_key = ''
client_secret = ''
access_token = ''
access_secret = ''
api = TwitterAPI(client_key, client_secret, access_token, access_secret)


# POLITIKEN.DK SEARCH #
SEARCH_TERM = 'url:"politiken.dk/del/"'
PRODUCT = '30day'
LABEL = 'prod'

r = api.request('tweets/search/%s/:%s' % (PRODUCT, LABEL), 
                {'query':SEARCH_TERM})

tweet_data = json.loads(r.text)
prettyjson = json.dumps(tweet_data, ensure_ascii=False, indent=4) # Only needed for debugging to pretify json

# Looks for all instances of expanded_url in json	
linklist = list(set(nested_lookup('expanded_url', tweet_data)))

urllist = []
for link in linklist:
	if "politiken.dk/del" in link:
		urllist.append(link)

# Request articles and get titles and dates and sort by dates
articlelist = []
titlecheck = []

for url in urllist:
	try:
		data = requests.get(url)
		result = data.text
		if '"isAccessibleForFree": "True"' not in result:
			soup = BeautifulSoup(result, "lxml")
			# Finds titles and timestamps
			title = soup.find('meta', attrs={'property':'og:title'})
			title = title['content']
			timestamp = soup.find('meta', attrs={'property':'article:published_time'})
			timestamp = timestamp['content']
			dateofarticle = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S%z')
			realurl = data.history[0].headers['Location']
			if title not in titlecheck:
				articlelist.append({"title": title, "url": realurl, "date": dateofarticle})
				titlecheck.append(title)			
	except:
		print(url)
			
articlelist_sorted = sorted(articlelist, key=lambda k: k['date'], reverse=True) 

# Check if article is already posted and update list of posted articles
with open("./pol_published.json", "r", encoding="utf8") as fin:
	alreadypublished = list(json.load(fin))
	for art in articlelist_sorted:
		url = art['url']
		token = url.index("?shareToken")
		url = url[:token]
		if url not in alreadypublished:
			alreadypublished.append(url)
			articlestopost.append(art)
	# Save updated already published links
	with open("./pol_published.json", "wt", encoding="utf8") as fout:
		alreadypublishedjson = json.dumps(alreadypublished)
		fout.write(alreadypublishedjson)

# ZETLAND.DK SEARCH #
SEARCH_TERM = 'url:"zetland.dk/historie"'
PRODUCT = '30day'
LABEL = 'prod'

r = api.request('tweets/search/%s/:%s' % (PRODUCT, LABEL), 
                {'query':SEARCH_TERM})

tweet_data = json.loads(r.text)
prettyjson = json.dumps(tweet_data, ensure_ascii=False, indent=4) # Only needed for debugging to pretify json

# Looks for all instances of expanded_url in json	
linklist = list(set(nested_lookup('expanded_url', tweet_data)))

urllist = []
for link in linklist:
	if "zetland.dk/historie" in link:
		urllist.append(link)

# Request articles and get titles and dates and sort by dates
articlelist = []
titlecheck = []

for url in urllist:
	try:
		data = requests.get(url)
		result = data.text

		# Soup site and create a dictionary of links and their titles and dates
		articledict = {}
		soup = BeautifulSoup(result, "lxml")

		title = soup.find('meta', attrs={'property':'og:title'})
		title = title['content']
		
		timestamp = soup.find('meta', attrs={'property':'article:published_time'})
		timestamp = timestamp['content']
		timestamp = timestamp[:timestamp.find("+")]
		dateofarticle = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f')
		
		if title not in titlecheck:
			articlelist.append({"title": title, "url": url, "date": dateofarticle})
			titlecheck.append(title)
	except:
		print(url)
			
articlelist_sorted = sorted(articlelist, key=lambda k: k['date'], reverse=True) 

# Check if article is already posted and update list of posted articles
with open("./zet_published.json", "r", encoding="utf8") as fin:
	alreadypublished = list(json.load(fin))
	for art in articlelist_sorted:
		title = art['title']
		if title not in alreadypublished:
			alreadypublished.append(title)
			articlestopost.append(art)
	# Save updated already published links
	with open("./zet_published.json", "wt", encoding="utf8") as fout:
		alreadypublishedjson = json.dumps(alreadypublished, ensure_ascii=False)
		fout.write(alreadypublishedjson)


# POST TO TWITTER #
if articlestopost:
	for art in articlestopost:
		if "zetland" in art['url']:
			medium = "Zetland"
		else:
			medium = "Politiken"
		status = "En flink abonnent på " + medium + " har delt en betalingsartikel. God fornøjelse! " + art['url']
		r = api.request('statuses/update', {'status': status})
		time.sleep(5)
Categories
blandet

Opdateret Python-program til at få E-boksbeskeder sendt på mail

Jeg har tidligere skrevet om mit Python-program til at få beskeder fra E-boks sendt på mail.

Nu har E-boks opgraderet sikkerheden, sådan man skal bruge Nemid til at aktivere E-boks på sin mobiltelefon, inden man får lov til at læse E-boks uden brug af Nemid.

Derfor var jeg nødt til at fikse mit program.

Løsningen blev at
bruge Android-emulatoren NOX, installerede app’en, aktivere med nemid og bruge Charles til at overvåge internettrafikken. Derefter genbrugte jeg app’ens kommunikation med E-boks’ server i mit program.

App’en sender en header med “deviceid” og en lang “challenge”-streng og xml-indhold med CPR-nummer og password for at logge på E-boks med “slevel 25”. Mon ikke “slevel” betyder security level?

I mit program kopierede jeg headeren og indholdet fra Charles ind min første kommunikation med E-boks (har redigeret noget ud):

content = '<?xml version="1.0" encoding="utf-8"?><Logon xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns="urn:eboks:mobile:1.0.0"><App version="3.6.1-MOBILEACCESS2.210" os="Android" osVersion="4.4.2" device="SM-G925F" /><User identity="[mit cpr-nummer]" identityType="P" nationality="DK" pincode="[mit password]" /></Logon>'

authstr = 'logon deviceid="[en id-streng]", datetime="2019-05-29 06:25:51Z", challenge="[en meget lang streng]"'

Det færdige program

Hvis du ønsker at bruge det færdige program, skal du:

  1. Installere Nox (eller en anden Android-emulator)
  2. Installere Charles (eller et andet program til at overvåge internettrafik)
  3. Opsætte Nox til at bruge Charles’ IP som proxy-server
  4. Åbne op for at kunne se SSL-traffik i Charles
  5. Installere E-boks-app’en i Nox og aktivere app’en med dit nemid
  6. Starte E-boks-app’en igen og logge på med dit CPR-nummer og password og logge trafikken i Charles
  7. Kopiere “content” og “authstr” fra Charles (se skærmbillede)og sætte ind i programmet hvor der står hhv. content = ” og authstr = ”

Held og lykke!

# -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com.
# Based on https://github.com/dk/Net-Eboks perl API for eboks.dk by Dmitry Karasik. Thanks!
""" This program logs on to e-boks.dk and takes new messages and sends them
to an e-mail. It requires prior Nemid activation using the E-boks app, e.g.
running the Nox emulator and a traffic monitor such as Charles. It also
requires access to a secure (SSL) SMTP server and mail account for sending e-mails. """

# Necessary modules
from datetime import datetime						# Current date and time
import requests										# Communicating with E-boks
import xml.etree.ElementTree as ET					# Parse E-boks XML responses
import smtplib										# Sending e-mails
from email.mime.multipart import MIMEMultipart		# Creating multipart e-mails
from email.mime.text import MIMEText				# Attaching text to e-mails
from email.mime.application import MIMEApplication	# Attaching pdf to e-mails
from email.mime.image import MIMEImage				# Attaching images to e-mails
from email.utils import formataddr					# Used for correct encoding of senders with special characters in name (e.g. Københavns Kommune)
import chardet										# Text message character set detection
import time											# Pause between e-mails sent

# Configuration data
data = {
'emailserver': '', 	# Your mail server hostname: host.server.dk
'emailserverport': ,					# Mail server port, e.g. 465
'emailusername': '',	# Sender mail account username
'emailpassword': '',		# Sender mail account password
'emailfrom': '',		# Sender e-mail, e.g. trump@usa.gov
'emailto': '',		# Recipient e-mail, e.g. hillary@clinton.net
'numberofmessagesperfolder': '10',		# Number of messages to request (10 is usually enough)
'unreadstatusvalue': "true",			# Normally "true". If "false" also read messages are sent
'unreadmorethan': 0,					# Normally 0, only unread messages are sent. If -1 all messages are sent
'sendemails': True,						# If True, e-mails are sent, if False, they are not
'country': 'DK',
'type': 'P',
'deviceid': '2',    # Use id from http traffic monitor
'datetime': '',
'root': 'rest.e-boks.dk',
'nonce': '',
'sessionid': '',
'response': '523b931af795698785df1eb85e8c10ea0687a46edb3a48468943f2d368fe725a',
'uid': '',
'uname': '',
'challenge': ''
}

# Gets current date and time for E-boks challenge
now = datetime.now()
data['datetime'] = datetime.strftime(now, '%Y-%m-%d %H:%M:%SZ')

# These functions are used to create sessionid, nonce and authstring values for communicating
# with E-boks throughout the program
def sessionid(authenticate):
	sessionstart = authenticate.find('sessionid="')+len('sessionid="')
	sessionend = authenticate.find('"', sessionstart)
	data['sessionid'] = authenticate[sessionstart:sessionend]

def nonce(authenticate):
	noncestart = authenticate.find('nonce="')+len('nonce="')
	nonceend = authenticate.find('"', noncestart)
	data['nonce'] = authenticate[noncestart:nonceend]

def createauthstring():
	authstr = 'deviceid="' + data['deviceid'] + '",nonce="' + data['nonce'] + ',sessionid="' + data['sessionid'] + '",response="' + data['response'] + '"'
	return authstr

# Logon to mail server
server = smtplib.SMTP_SSL(data['emailserver'], data['emailserverport'])
server.login(data['emailusername'], data['emailpassword'])
	
# First logon to e-boks
url = "https://" + data['root'] + "/mobile/1/xml.svc/en-gb/session"

# Use XML string from http traffic monitor
content = ''

# Use string from http traffic monitor
authstr = ''

headers = {
'Content-Type': 'application/xml',
'Content-Length': str(len(content)),
'X-EBOKS-AUTHENTICATE': authstr,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Accept-Encoding': 'gzip,deflate',
'Host': data['root'],
}

r = requests.put(url, headers=headers, data=content)

authenticate = r.headers['X-EBOKS-AUTHENTICATE']
nonce(authenticate)
sessionid(authenticate)

xml = ET.fromstring(r.text)
# Saves username and user id
data['uname'] = xml[0].attrib['name']
data['uid'] = xml[0].attrib['userId']

# Get folder data from e-boks
url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folders'
authstr = createauthstring()

headers = {
'X-EBOKS-AUTHENTICATE': authstr,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Host': data['root'],
}

r = requests.get(url, headers=headers)
authenticate = r.headers['X-EBOKS-AUTHENTICATE']
nonce(authenticate)

xml = ET.fromstring(r.text)

eboks_folders = xml

# Get folder id's and numbers of unread messages
for folder in eboks_folders:
	folderid = folder.attrib['id']
	unread = folder.attrib['unread']
	
	# Get messages ONLY if any unread messages in folder
	if int(unread) > data['unreadmorethan']:		# Usually > 0. Can be changed to == 0 for debugging purposes
		
		# Get list of messages
		url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid
		
		authstr = createauthstring()

		headers = {
		'X-EBOKS-AUTHENTICATE': authstr,
		'Accept': '*/*',
		'Accept-Language': 'en-US',
		'Host': data['root'],
		}

		params = {
		'skip': '0',
		'take': data['numberofmessagesperfolder']
		}

		r = requests.get(url, headers=headers, params=params)
		authenticate = r.headers['X-EBOKS-AUTHENTICATE']
		nonce(authenticate)		
		
		xml = ET.fromstring(r.text)
		eboks_messages = xml

		i = 0
		max = int(params['take']) - 1
		while i <= max:
			for child in eboks_messages:
				messageid = child[i].attrib['id']
				subject  = child[i].attrib['name']
				sender = child[i][0].text
				unreadstatus = child[i].attrib['unread']
				attachmentcount = child[i].attrib['attachmentsCount']
				format = child[i].attrib['format'].lower()
				received = child[i].attrib['receivedDateTime']
				
				i += 1
				
				# Get only messages that are unread
				if unreadstatus == data['unreadstatusvalue']:	# Usually true. Can be changed to false for debugging purposes
					
					# Start e-mail
					msg = MIMEMultipart()
					msg['From'] = formataddr((sender, data['emailfrom']))
					msg['To'] = data['emailto']
					msg['Subject'] = "E-boks: " + subject
					body = ""
					
										
					# Get message (marks it as read)
					url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + messageid

					authstr = createauthstring()

					headers = {
					'X-EBOKS-AUTHENTICATE': authstr,
					'Accept': '*/*',
					'Accept-Language': 'en-US',
					'Host': data['root'],
					}

					r = requests.get(url, headers=headers)
					authenticate = r.headers['X-EBOKS-AUTHENTICATE']
					nonce(authenticate)
					
					# Get primary message content
					url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + messageid + '/content'

					authstr = createauthstring()

					headers = {
					'X-EBOKS-AUTHENTICATE': authstr,
					'Accept': '*/*',
					'Accept-Language': 'en-US',
					'Host': data['root'],
					}

					r = requests.get(url, headers=headers)
					authenticate = r.headers['X-EBOKS-AUTHENTICATE']
					nonce(authenticate)
					
					# Attach primary message content to e-mail
					if format in ("txt","text","plain"):
						characterset = chardet.detect(r.content)
						r.encoding = characterset['encoding']
						body = r.text
						msg.attach(MIMEText(body, 'plain'))
					elif format in ("html","htm"):
						characterset = chardet.detect(r.content)
						r.encoding = characterset['encoding']
						body = r.text
						msg.attach(MIMEText(body, 'html'))						
					elif format == "pdf":
						filename = "".join().rstrip() + "." + format
						part = MIMEApplication(r.content)
						part.add_header('Content-Disposition', 'attachment', filename = filename)
						msg.attach(part)
					elif format in ("gif","jpg","jpeg","tiff","tif","webp"):
						filename = "".join().rstrip() + "." + format
						part = MIMEImage(r.content)
						part.add_header('Content-Disposition', 'attachment', filename = filename)
						msg.attach(part)
										
					# Get attachment data if message has attachments
					if int(attachmentcount) > 0:

						url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + messageid

						authstr = createauthstring()

						headers = {
						'X-EBOKS-AUTHENTICATE': authstr,
						'Accept': '*/*',
						'Accept-Language': 'en-US',
						'Host': data['root'],
						}

						r = requests.get(url, headers=headers)
						authenticate = r.headers['X-EBOKS-AUTHENTICATE']
						nonce(authenticate)

						xml = ET.fromstring(r.text)
						
						eboks_attachment = xml
						
						# Gets if, name and format of attachment
						for child in eboks_attachment:
							for subtree in child:
								attachmentid = subtree.attrib['id']
								attachmenttitle = subtree.attrib['name']
								attachmentformat = subtree.attrib['format']
				
								# Gets the actual attachment
								url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + attachmentid + '/content'
									
								authstr = createauthstring()

								headers = {
								'X-EBOKS-AUTHENTICATE': authstr,
								'Accept': '*/*',
								'Accept-Language': 'en-US',
								'Host': data['root'],
								}

								r = requests.get(url, headers=headers)
								authenticate = r.headers['X-EBOKS-AUTHENTICATE']
								nonce(authenticate)
								
								# Attach attachment to e-mail
								if attachmentformat in ("txt","text","html","htm","plain"):
									filename = "".join().rstrip() + "." + attachmentformat
									r.encoding = "utf-8"
									part = MIMEText(r.text)
									part.add_header('Content-Disposition', 'attachment', filename = filename)
									msg.attach(part)
								elif attachmentformat == "pdf":
									filename = "".join().rstrip() + "." + attachmentformat
									part = MIMEApplication(r.content)
									part.add_header('Content-Disposition', 'attachment', filename = filename)
									msg.attach(part)
								elif attachmentformat in ("gif","jpg","jpeg","tiff","tif","webp"):
									filename = "".join().rstrip() + "." + attachmentformat
									part = MIMEImage(r.content)
									part.add_header('Content-Disposition', 'attachment', filename = filename)
									msg.attach(part)

					# Send e-mail
					if data['sendemails'] == True:
						print("sending")
						msg.attach(MIMEText(body, 'plain'))
						server.sendmail(data['emailfrom'], data['emailto'], msg.as_string())
						time.sleep(2)

Categories
blandet

Sådan låner jeg bøger på eReolen – med Python

eReolen er fint til at læse gratis e-bøger på, men både hjemmesiden og app’en synes jeg er kluntet at bruge. Derfor har jeg undersøgt, om jeg kunne bruge Python til at snakke med eReolen – og så udelukkende bruge app’en til at læse på.

Først havde jeg brug for at finde ud af, hvordan eReolens app virker.

Jeg installerede Android-emulatoren NOX og installerede app’en. Derefter hentede jeg Charles, som er et værktøj til at overvåge internettraffik. Jeg satte NOX op til at bruge Charles’ IP som proxyserver, åbnede op i Charles for at kunne se krypteret trafik og vupti – nu kunne jeg se trafikken fra NOX i Charles.

Det ser nogenlunde sådan ud, når man logger på eReolen:

eReolens app snakker altså med ereolen-be.redia.dk og en side på serveren, der hedder rpc.php. I spalten “Info” kan jeg se forskellige funktioner, som rpc.php understøtter. Den allerførste (authenticate) er login. Den kiggede jeg nærmere på:

Browseren sender data af sted i JSON-format
JSON-strengen har et id, som ser ud til at være tilfældigt genereret, fortæller hvilken protokol, der bruges, en metode (her “authenticate”) og nogle parametre (bibliotekskode, CPR-nummer og PIN-kode)

Når det lykkes at logge på, får man nogenlunde dette data tilbage:

{
	"jsonrpc": "2.0",
	"id": "[slettet]",
	"result": {
		"result": true,
		"data": {
			"retailerid": "810",
			"cardnumber": "[slettet]",
			"ereolen": {
				"result": true,
				"data": null,
				"message": "",
				"language": "da",
				"code": 0
			},
			"netlydbog": {
				"result": true,
				"data": null,
				"message": "",
				"language": "da",
				"code": 0
			}
		},
		"message": "",
		"language": "da",
		"code": 0
	}
}

Code 0 betyder, at login er lykkedes.

Tilbage får man også en cookie med et ID for den “session”, man har startet. Den hedder PHPSESSID.

Sådan her ser login-proceduren ud i Python:

import requests		# Used for http requests
import uuid			# Used to generate random strings for json request ids

# Credentials
# Library ID as string (See end of program for list), CPR-number as string (1234561234), PIN as string
logininfo = ['810','1234567890','1234']

# Login and get session cookie
headers = {
	'Accept-Charset': 'UTF-8',
	'Content-Type':	'application/json',
	'User-Agent': 'Dalvik/1.6.0 (Linux; U; Android 4.4.2; SM-G925F Build/JLS36C)',
	'Accept-Encoding': 'gzip'
}	

json = {
	"id": str(uuid.uuid4()),
	"jsonrpc": "2.0",
	"method": "authenticate",
	"params": logininfo
}

url = 'https://ereolen-be.redia.dk/v5/rpc.php'
r = requests.post(url, headers=headers, json=json)
loginresponse = r.json()

# Save session cookie to cookie dictionary
cookies = {}
cookies['PHPSESSID'] = r.cookies['PHPSESSID']
cookies['ROUTEID'] = r.cookies['ROUTEID']

Det er let at undersøge de forskellige funktioner/metoder i Charles. I dette tilfælde var jeg interesseret i at låne en bog. Den funktion hedder, logisk nok, createLoan.

Her er hvad eReolens app sender af sted, når man låner en ebog:

{
	"id": "[slettet]",
	"jsonrpc": "2.0",
	"method": "createLoan",
	"params": ["9788792922021", "ereolen"]
}

En søgning på tallet i “params” viser, at vi her har at gøre med Håbløse slægter af Herman Bang. Tallet er simpelthen bogens ISBN-nummer.

Det færdige program

Det færdige program, der lader låneren indtaste et ISBN-nummer på en bog, som så bliver lånt, ser sådan her ud.

Du er velkommen til at bruge programmet, videreudvikle, og hvad du ellers har lyst til.

# -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com
""" Loan a book at eReolen """

import requests		# Used for http requests
import uuid			# Used to generate random strings for json request ids

# Credentials
# Library ID as string (See end of program for list), CPR-number as string (1234561234), PIN as string
logininfo = ['810','1234567890','1234']

# Login and get session cookie
headers = {
	'Accept-Charset': 'UTF-8',
	'Content-Type':	'application/json',
	'User-Agent': 'Dalvik/1.6.0 (Linux; U; Android 4.4.2; SM-G925F Build/JLS36C)',
	'Accept-Encoding': 'gzip'
}	

json = {
	"id": str(uuid.uuid4()),
	"jsonrpc": "2.0",
	"method": "authenticate",
	"params": logininfo
}

url = 'https://ereolen-be.redia.dk/v5/rpc.php'
r = requests.post(url, headers=headers, json=json)
loginresponse = r.json()

# Save session cookie to cookie dictionary
cookies = {}
cookies['PHPSESSID'] = r.cookies['PHPSESSID']
cookies['ROUTEID'] = r.cookies['ROUTEID']

# Check for successful login
if loginresponse["result"]["result"] == True and loginresponse["result"]["data"]["ereolen"]["code"] == 0:
	print("Login lykkedes")
	print("")
	validinput = False
	# Loop to ensure the user enters valid ISBN
	while validinput ==  False:
		isbn = input("Indtast ISBN (13 cifre) på den bog, du vil låne: ")
		# ISBN must be 13 characters and digits only
		if len(isbn) == 13 and isbn.isdigit():
			validinput = True
			isbn = str(isbn)
			# Create loan
			json = {
				"id": str(uuid.uuid4()),
				"jsonrpc": "2.0",
				"method": "createLoan",
				"params": [isbn, "ereolen"]
			}
			r = requests.post(url, headers=headers, cookies=cookies, json=json)
			loanresponse = r.json()
			# Check for loan success
			if loanresponse["result"]["code"] == 0:
				print("Tillykke! Det lykkes at låne bogen. God fornøjelse.")
			else:
				print("Noget gik galt. Måske kan dette gøre dig klogere:")
				print("")
				print(loanresponse)
				print("")
		else:
			print("")
			print("Det ligner, du har tastet forkert ISBN. Prøv igen.")
			print("")
else:
	print("Noget gik galt ved login. Prøv at tjekke dine logininformationer.")

#LIBRARY IDS
'''
Haderslev:851
Frederiksberg:823
Herlev:866
Hedensted:912
Brøndby:860
Lolland:874
Esbjerg:826
Guldborgsund:873
Brønderslev:886
Vesthimmerland:890
Ikast-Brande:835
Glostrup:856
Assens:953
Roskilde:824
Hjørring:867
Køge:872
Greve:830
Samsø:936
Rebild:862
Frederikshavn:880
Dragør:905
Høje-Taastrup:875
Thisted:885
Nordfyn:950
Svendborg:897
Aarhus:816
Langeland:852
Herning:828
Vallensbæk:877
Favrskov:888
Kerteminde:917
Skive:907
Holbæk:899
Hillerød:861
Halsnæs:887
Jammerbugt:902
Ballerup:843
Kalundborg:910
Helsingør:896
Varde:834
Odsherred:879
Viborg:854
Stevns:882
Fredensborg:855
Fredericia:891
Sorø:848
Frederikssund:870
Faxe:895
Norddjurs:906
Rødovre:838
Randers:916
København:810
Hvidovre:853
Sydslesvig:889
Slagelse:894
Tønder:900
Ringsted:884
Albertslund:881
Silkeborg:829
Lejre:935
Gentofte:832
Odense:825
Struer:869
Faaborg-Midtfyn:840
Vejen:893
Ærø:954
Rudersdal:841
Hørsholm:871
Middelfart:864
Billund:850
Allerød:952
Solrød:844
Egedal:898
Lemvig:911
Býarbókasavnið:955
Horsens:847
Kolding:878
Gladsaxe:863
Aabenraa:839
Odder:876
Gribskov:903
Nyborg:858
Ishøj:868
Vejle:849
Sønderborg:836
Mariagerfjord:842
Skanderborg:883
Vordingborg:901
Morsø:904
Lyngby:857
Syddjurs:892
Aalborg:822
Bornholm:865
Furesø:908
Holstebro:859
Næstved:831
Tårnby:837
Ringkøbing-Skjern:845
'''

Categories
blandet

Sådan trækker du dine data ud fra Saxo Bank med Python

Opdateret d. 20/10/2019: Nogle gange har Saxo Bank en “disclaimer” (i dette tilfælde omkring Brexit), som de vil vise, inden man får lov at logge på. Jeg har tilpasset koden, sådan programmet kan håndtere dette tilfælde.

Jeg har tidligere skrevet om, hvordan jeg trækker transaktionsdata ud fra mine konti hos Nordnet.

Nu er jeg også blevet kunde hos Saxo Bank. (Hvorfor? Mulighed for at oprette en aktiesparekonto og ingen minimumskurtage.)

Selvbetjeningsløsningen hos Saxo Bank er rigtig dårlig (sammenlignet med Nordnets).

Derfor var jeg interesseret i, om jeg kunne finde en måde at trække transaktionsdata ud fra min konto hos Saxo Bank – uden at have brug for at se på hjemmesiden.

Det kunne jeg. Måske ikke på den smarteste måde i verden, for Saxo Bank har faktisk en API-løsning, man kan bruge, hvis man har mod på at udfylde en aftale i hånden og scanne den ind (det gad jeg ikke).

Her kan du læse, hvordan jeg fik fat i mine data.

Snakke http med Saxo

Ligesom da jeg hentede mine transaktioner hos Nordnet, undersøgte jeg, hvordan min browser snakker med – og viser data fra – min konto hos Saxo Bank.

Allerførst kiggede jeg på, hvad hjemmesiden gør, når den skal vise mine data. Jeg valgte gennemførte handler i menuen:

Og så kiggede jeg på, hvad browseren gjorde. Det viser sig at hjemmesiden – fornuftigt nok – bruger Saxo Banks API, når den skal vise data til brugeren:

Jeg kunne se, at API’et modtog en lang streng (“Authorization” med ordet BEARER foran). Den gik jeg ud fra, var nødvendig, for at få data tilbage.

Så spørgsmålet var egentlig bare: Hvordan bliver sådan en BEARER-streng genereret?

Tilbage til start

For at komme frem til, hvordan BEARER-strengen genereres, gik jeg tilbage til start: Jeg gik til loginsiden og trykkede F12 i min browser (Chrome) for at følge med i netværksforespørgslerne.

Loginsidens formular sender mit brugernavn og password af sted, sammen med en streng – “AuthnRequest” – der genereres på ny hver gang, loginsiden hentes:

I mit Python-program prøver jeg at sende sådan en formular af sted, og undersøger hvad jeg får tilbage.

# Visit login page and get AuthnRequest token value from input form
url = 'https://www.saxoinvestor.dk/Login/da/'
r = requests.get(url)

soup = BeautifulSoup(r.text, "html.parser")
input = soup.find_all('input', {"id":"AuthnRequest"})
authnrequest = input[0]["value"]

# Login step 1: Submit username, password and token and get another token back
url = 'https://www.saxoinvestor.dk/Login/da/'
r = requests.post(url, data = {'field_userid': user, 'field_password': password, 'AuthnRequest': authnrequest})

Lidt forkortet ser det sådan her ud:

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta name="Application-State" content="service=IDP;federated=False;env=Live;state=Ok;authenticated=True;"><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
</head>
<body>
<noscript><p><strong>Note:</strong> Since your browser does not support Javascript, you must press the Continue button to proceed.</p></noscript>
<form id="form" action="https://www.saxoinvestor.dk/investor/login.sso.ashx" method="post"><div>
<input type="hidden" name="SAMLResponse" value="PHNhbWxwOlJlc3BvbnNlIElEPSJfNjQzZGI4ODQtMDMzZi00MWVhLWE4ZjEtYzVjOWVlMWIxM2IwIiBJblJlc3BvbnNlVG89Il9mN2E3ODBlZi0yZjdmLTQyYmItODk1[...]G9uc2U+"/>
<input type="hidden" name="RelayState" value=""/>
<input type="hidden" name="PageLoadInfo" id="PageLoadInfo" value=""/></div>
<noscript><div>
<input type="submit" value="Continue"/></div></noscript></form><script type="text/javascript">function doSubmit(){var t=-1;if(window.location.hash){var m=window.location.hash.match(/\/lst\/(\d+)/);if(m) t=parseInt(m[1]);}if(t>=0 &amp;&amp; document.getElementById("PageLoadInfo").value=='')document.getElementById("PageLoadInfo").value=t;document.forms.form.submit();}</script><script  type="text/javascript">doSubmit();</script></body></html>

Og hvad er det så? En formular (“<input>”) til browsere uden Javascript, med et felt der hedder “SAMLResponse” med en lang (her forkortet) streng som værdi.

I Chrome kan jeg se, at allersidste trin. inden jeg når ind på forsiden af selvbetjeningen, faktisk er, at min browser sender “SAMLResponse” af sted til en side, der hedder “login.sso.ashx”:

Så jeg sender trygt formularen af sted med SAMLResponse-.værdien og ser, hvad jeg får tilbage:

soup = BeautifulSoup(r.text, "html.parser")
input = soup.find_all('input', {"name":"SAMLResponse"})
samlresponse = input[0]["value"]

# Login step 2: Get bearer token necessary for API requests
url = 'https://www.saxoinvestor.dk/investor/login.sso.ashx'
r = requests.post(url, data = {'SAMLResponse': samlresponse})

Og vupti: Siden videresender mig til API’et med et BEARER-token, jeg kan benytte mig af. Det får jeg fat i (og skærer lidt til) sådan her::

bearer = r.history[0].headers['Location']
bearer = bearer[bearer.find("BEARER"):bearer.find("/exp/")]
bearer = bearer.replace("%20"," ")

Og så er jeg ellers klar til at hente data fra API’et. Det starter sådan her:

# START API CALLS
# Documentation at https://www.developer.saxo/openapi/learn

# Set bearer token as header
headers = {'Authorization': bearer}

# First API request gets Client Key which is used for most API calls
# See https://www.developer.saxo/openapi/learn/the-tutorial for expected return data
url = 'https://www.saxoinvestor.dk/openapi/port/v1/clients/me'
r = requests.get(url, headers=headers)

clientdata = r.json()
clientkey = clientdata['ClientKey']

Hele programmet

Hele programmet – inklusive den måde, jeg omdanner Saxo Bank-data til samme format som Nordnets transaktionsdata – er her.

Du må gøre præcis som du har lyst til med det (på eget ansvar).

# -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com
"""This program logs into a Saxo Bank account and lets you make API requests."""

import requests 
from datetime import datetime
from datetime import date
from bs4 import BeautifulSoup
import json

# USER ACCOUNT AND PERIOD DATA. SHOULD BE EDITED FOR YOUR NEEDS #

# Saxo user account credentials
user = '' # your user id
password = '' # your password

# Start date (start of period for transactions) and date today used for extraction of transactions
startdate = '2019-01-01'
today = date.today()
enddate = datetime.strftime(today, '%Y-%m-%d')

# LOGIN TO SAXO BANK
	
# Visit login page and get AuthnRequest token value from input form
url = 'https://www.saxoinvestor.dk/Login/da/'
r = requests.get(url)

soup = BeautifulSoup(r.text, "html.parser")
input = soup.find_all('input', {"id":"AuthnRequest"})
authnrequest = input[0]["value"]

# Login step 1: Submit username, password and token and get another token back
url = 'https://www.saxoinvestor.dk/Login/da/'
r = requests.post(url, data = {'field_userid': user, 'field_password': password, 'AuthnRequest': authnrequest})

soup = BeautifulSoup(r.text, "html.parser")
input = soup.find_all('input', {"name":"SAMLResponse"})
# Most of the time this works
if input:
	samlresponse = input[0]["value"]
# But sometimes there's a disclaimer that Saxo Bank would like you to accept
else:
	input = soup.find_all('input')
	inputs = {}
	try:
		for i in input:
			inputs[i['name']] = i['value']
	except:
		pass
	url = 'https://www.saxotrader.com/disclaimer'
	request = requests.post(url, data=inputs)
	cook = request.cookies['DisclaimerApp']
	returnurl = cook[cook.find("ReturnUrl")+10:cook.find("&amp;IsClientStation")]
	url = 'https://live.logonvalidation.net/complete-app-consent/' + returnurl[returnurl.find("complete-app-consent/")+21:]
	request = requests.get(url)
	soup = BeautifulSoup(request.text, "html.parser")
	input = soup.find_all('input', {"name":"SAMLResponse"})
	samlresponse = input[0]["value"]

# Login step 2: Get bearer token necessary for API requests
url = 'https://www.saxoinvestor.dk/investor/login.sso.ashx'
r = requests.post(url, data = {'SAMLResponse': samlresponse})

bearer = r.history[0].headers['Location']
bearer = bearer[bearer.find("BEARER"):bearer.find("/exp/")]
bearer = bearer.replace("%20"," ")

# START API CALLS
# Documentation at https://www.developer.saxo/openapi/learn

# Set bearer token as header
headers = {'Authorization': bearer}

# First API request gets Client Key which is used for most API calls
# See https://www.developer.saxo/openapi/learn/the-tutorial for expected return data
url = 'https://www.saxoinvestor.dk/openapi/port/v1/clients/me'
r = requests.get(url, headers=headers)

clientdata = r.json()
clientkey = clientdata['ClientKey']

# Example API call #1
url = 'https://www.saxoinvestor.dk/openapi/cs/v1/reports/aggregatedAmounts/' + clientkey + '/' + startdate + '/' + enddate + '/'
r = requests.get(url, headers=headers)
data = r.json()

# Working on that data to add some transaction types to personal system
saxoaccountname = "Aktiesparekonto: Saxo Bank"
currency = "DKK"
saxotransactions = ""

for item in data['Data']:
	if item['AffectsBalance'] == True:
		date = item['Date']
		amount = item['Amount']
		amount_str = str(amount).replace(".",",")
		if item['UnderlyingInstrumentDescription'] == 'Cash deposit or withdrawal' or item['UnderlyingInstrumentDescription'] == 'Cash inter-account transfer':
			if amount > 0:
				transactiontype = 'INDBETALING'
			elif amount < 0:
				transactiontype = 'HÆVNING'
			saxotransactions += ";" + date + ";" + date + ";" + date + ";" + transactiontype + ";;;;;;;;" + amount_str + ";" + currency + ";;;;;;;;;" + saxoaccountname + "\r\n"
		if item['AmountTypeName'] == 'Corporate Actions - Cash Dividends':
			transactiontype = "UDB."
			if item['InstrumentDescription'] == "Novo Nordisk B A/S":
				paper = "Novo B"
				papertype = "Aktie"
			if item['InstrumentDescription'] == "Tryg A/S":
				paper = "TRYG"
				papertype = "Aktie"
			saxotransactions += ";" + date + ";" + date + ";" + date + ";" + transactiontype + ";" + paper + ";" + papertype + ";;;;;;" + amount_str + ";" + currency + ";;;;;;;;;" + saxoaccountname + "\n"

# Example API call #2		
url = "https://www.saxoinvestor.dk/openapi/cs/v1/reports/trades/" + clientkey + "?fromDate=" + startdate + "&amp;" + "toDate=" + enddate
r = requests.get(url, headers=headers)
data = r.json()

# Working on that data to add trades to personal system
for item in data['Data']:
	date = item['AdjustedTradeDate']
	numberofpapers = str(int(item['Amount']))
	amount_str = str(item['BookedAmountAccountCurrency']).replace(".",",")
	priceperpaper = str(item['BookedAmountAccountCurrency'] / item['Amount']).replace(".",",")
	if item['TradeEventType'] == 'Bought':
		transactiontype = "KØBT"
	if item['AssetType'] == 'Stock':
		papertype = "Aktie"
	if item['InstrumentDescription'] == "Novo Nordisk B A/S":
		paper = "Novo B"
		isin = "DK0060534915"
	if item['InstrumentDescription'] == "Tryg A/S":
		paper = "TRYG"
		isin = "DK0060636678"
	saxotransactions += ";" + date + ";" + date + ";" + date + ";" + transactiontype + ";" + paper + ";" + papertype + ";" + isin + ";" + numberofpapers + ";" + priceperpaper + ";;;" + amount_str + ";" + currency + ";;;;;;;;;" + saxoaccountname + "\n"
Categories
blandet

Python-program til at sende dine beskeder i E-boks til din mailadresse

OPDATERING 29. maj 2019: Dette program virker ikke længere. Prøv dette i stedet.

Der er mange, der har fået den idé, automatisk at videresende beskeder i E-boks til en anden mailadresse eller gemme dem på computeren. Videresendelse kan også gøres i E-boks selv, men kun manuelt og en besked ad gangen. Her er nogle projekter:

Jeg kan kun finde ud af Python, og ved hvordan jeg sætter et job op på min webserver, der kan gøre Python-programmer regelmæssigt (det skulle jeg nemlig bruge til https://wallnot.dk).

Så – med kæmpe hjælp fra koden til Net-Eboks af Dmitry Karasik – har jeg skrevet et Python-program, der videresender nye beskeder i E-boks til min mail.

Programmet fungerer, men det er ikke gennemtestet, og tager ikke højde for fejl, fx. at brugeren indtaster forkerte oplysninger i programmet. Så det er nok en god idé også at logge ind på E-boks en gang imellem og lige se at alt bliver hentet og videresendt.

Du er velkommen til at bruge programmet, videreudvikle, og hvad du ellers kan finde på.

# -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com.
# Based on https://github.com/dk/Net-Eboks perl API for eboks.dk by Dmitry Karasik. Thanks!
""" This program logs on to e-boks.dk and takes new messages and sends them
to an e-mail. It requires mobile app login for e-boks (see http://www.e-boks.dk/help.aspx?pageid=db5a89a1-8530-418a-90e9-ff7f0713784a for
how to create). It also requires access to a secure (SSL) SMTP server and mail 
account for sending e-mails. """

# Necessary modules
from datetime import datetime						# Current date and time
import requests										# Communicating with E-boks
import hashlib										# Hash configuration for challenge/logon
import xml.etree.ElementTree as ET					# Parse E-boks XML responses
import smtplib										# Sending e-mails
from email.mime.multipart import MIMEMultipart		# Creating multipart e-mails
from email.mime.text import MIMEText				# Attaching text to e-mails
from email.mime.application import MIMEApplication	# Attaching pdf to e-mails
from email.mime.image import MIMEImage				# Attaching images to e-mails
from email.utils import formataddr					# Used for correct encoding of senders with special characters in name (e.g. Københavns Kommune)
import chardet										# Text message character set detection
import time											# Pause between e-mails sent

# Configuration data
data = {
'emailserver': '', 	# Your mail server hostname: host.server.dk
'emailserverport': ,					# Mail server port, e.g. 465
'emailusername': '',	# Sender mail account username
'emailpassword': '',		# Sender mail account password
'emailfrom': '',		# Sender e-mail, e.g. trump@usa.gov
'emailto': '',		# Recipient e-mail, e.g. hillary@clinton.net
'cpr': '',					# CPR number (no hyphens), e.g. 1234567890
'password': '',		# E-boks mobile account password
'activation': '',				# E-boks mobile account activation code
'numberofmessagesperfolder': '10',		# Number of messages to request (10 is usually enough)
'unreadstatusvalue': "true",			# Normally "true". If "false" also read messages are sent
'unreadmorethan': 0,					# Normally 0, only unread messages are sent. If -1 all messages are sent
'sendemails': True,						# If True, e-mails are sent, if False, they are not
'country': 'DK',
'type': 'P',
'deviceid': 'python-e-boks-000000000000',
'datetime': '',
'root': 'rest.e-boks.dk',
'nonce': '',
'sessionid': '',
'response': '3a1a51f235a8bd6bbc29b2caef986a1aeb77018d60ffdad9c5e31117e7b6ead3',
'uid': '',
'uname': '',
'challenge': ''
}

# Gets current date and time for E-boks challenge
now = datetime.now()
data['datetime'] = datetime.strftime(now, '%Y-%m-%d %H:%M:%SZ')

# Hashes parts of configuration data and sets challenge value to authenticate with E-boks
hashstring = data['activation']+":"+data['deviceid']+":"+data['type']+":"+data['cpr']+":"+data['country']+":"+data['password']+":"+data['datetime']
hashstringcoded = hashstring.encode('utf-8')
data['challenge'] = hashlib.sha256(hashstringcoded).hexdigest().encode('utf-8')
data['challenge'] = hashlib.sha256(data['challenge']).hexdigest()

# These functions are used to create sessionid, nonce and authstring values for communicating
# with E-boks throughout the program
def sessionid(authenticate):
	sessionstart = authenticate.find('sessionid="')+len('sessionid="')
	sessionend = authenticate.find('"', sessionstart)
	data['sessionid'] = authenticate[sessionstart:sessionend]

def nonce(authenticate):
	noncestart = authenticate.find('nonce="')+len('nonce="')
	nonceend = authenticate.find('"', noncestart)
	data['nonce'] = authenticate[noncestart:nonceend]

def createauthstring():
	authstr = 'deviceid="' + data['deviceid'] + '",nonce="' + data['nonce'] + ',sessionid="' + data['sessionid'] + '",response="' + data['response'] + '"'
	return authstr

# Logon to mail server
server = smtplib.SMTP_SSL(data['emailserver'], data['emailserverport'])
server.login(data['emailusername'], data['emailpassword'])
	
# First logon to e-boks
url = "https://" + data['root'] + "/mobile/1/xml.svc/en-gb/session"

content = '<Logon xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns="urn:eboks:mobile:1.0.0"><User identity="' + data['cpr'] + '" identityType="' + data['type'] + '" nationality="' + data['country'] + '" pincode="' + data['password'] + '"/></Logon>'

authstr = 'logon ' + 'deviceid="' + data['deviceid']+ '",' + 'datetime="' + data['datetime'] + '",' + 'challenge="' + data['challenge'] + '"'

headers = {
'Content-Type': 'application/xml',
'Content-Length': str(len(content)),
'X-EBOKS-AUTHENTICATE': authstr,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Accept-Encoding': 'gzip,deflate',
'Host': data['root'],
}

r = requests.put(url, headers=headers, data=content)

authenticate = r.headers['X-EBOKS-AUTHENTICATE']
nonce(authenticate)
sessionid(authenticate)

xml = ET.fromstring(r.text)
# Saves username and user id
data['uname'] = xml[0].attrib['name']
data['uid'] = xml[0].attrib['userId']

# Get folder data from e-boks
url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folders'
authstr = createauthstring()

headers = {
'X-EBOKS-AUTHENTICATE': authstr,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Host': data['root'],
}

r = requests.get(url, headers=headers)
authenticate = r.headers['X-EBOKS-AUTHENTICATE']
nonce(authenticate)

xml = ET.fromstring(r.text)

eboks_folders = xml

# Get folder id's and numbers of unread messages
for folder in eboks_folders:
	folderid = folder.attrib['id']
	unread = folder.attrib['unread']
	
	# Get messages ONLY if any unread messages in folder
	if int(unread) > data['unreadmorethan']:		# Usually > 0. Can be changed to == 0 for debugging purposes
		
		# Get list of messages
		url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid

		authstr = createauthstring()

		headers = {
		'X-EBOKS-AUTHENTICATE': authstr,
		'Accept': '*/*',
		'Accept-Language': 'en-US',
		'Host': data['root'],
		}

		params = {
		'skip': '0',
		'take': data['numberofmessagesperfolder']
		}

		r = requests.get(url, headers=headers, params=params)
		authenticate = r.headers['X-EBOKS-AUTHENTICATE']
		nonce(authenticate)		
		
		xml = ET.fromstring(r.text)
		
		eboks_messages = xml
		
		i = 0
		max = int(params['take']) - 1
		
		while i  <= max:
			for child in eboks_messages:
				messageid = child[i].attrib['id']
				subject  = child[i].attrib['name']
				sender = child[i][0].text
				unreadstatus = child[i].attrib['unread']
				attachmentcount = child[i].attrib['attachmentsCount']
				format = child[i].attrib['format'].lower()
				received = child[i].attrib['receivedDateTime']
				
				i += 1
				
				# Get only messages that are unread
				if unreadstatus == data['unreadstatusvalue']:	# Usually true. Can be changed to false for debugging purposes
					
					# Start e-mail
					msg = MIMEMultipart()
					msg['From'] = formataddr((sender, data['emailfrom']))
					msg['To'] = data['emailto']
					msg['Subject'] = "E-boks: " + subject
					body = ""
					
										
					# Get message (marks it as read)
					url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + messageid

					authstr = createauthstring()

					headers = {
					'X-EBOKS-AUTHENTICATE': authstr,
					'Accept': '*/*',
					'Accept-Language': 'en-US',
					'Host': data['root'],
					}

					r = requests.get(url, headers=headers)
					authenticate = r.headers['X-EBOKS-AUTHENTICATE']
					nonce(authenticate)
					
					# Get primary message content
					url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + messageid + '/content'

					authstr = createauthstring()

					headers = {
					'X-EBOKS-AUTHENTICATE': authstr,
					'Accept': '*/*',
					'Accept-Language': 'en-US',
					'Host': data['root'],
					}

					r = requests.get(url, headers=headers)
					authenticate = r.headers['X-EBOKS-AUTHENTICATE']
					nonce(authenticate)
					
					# Attach primary message content to e-mail
					if format in ("txt","text","plain"):
						characterset = chardet.detect(r.content)
						r.encoding = characterset['encoding']
						body = r.text
						msg.attach(MIMEText(body, 'plain'))
					elif format in ("html","htm"):
						characterset = chardet.detect(r.content)
						r.encoding = characterset['encoding']
						body = r.text
						msg.attach(MIMEText(body, 'html'))						
					elif format == "pdf":
						filename = "".join().rstrip() + "." + format
						part = MIMEApplication(r.content)
						part.add_header('Content-Disposition', 'attachment', filename = filename)
						msg.attach(part)
					elif format in ("gif","jpg","jpeg","tiff","tif","webp"):
						filename = "".join().rstrip() + "." + format
						part = MIMEImage(r.content)
						part.add_header('Content-Disposition', 'attachment', filename = filename)
						msg.attach(part)
										
					# Get attachment data if message has attachments
					if int(attachmentcount) > 0:

						url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + messageid

						authstr = createauthstring()

						headers = {
						'X-EBOKS-AUTHENTICATE': authstr,
						'Accept': '*/*',
						'Accept-Language': 'en-US',
						'Host': data['root'],
						}

						r = requests.get(url, headers=headers)
						authenticate = r.headers['X-EBOKS-AUTHENTICATE']
						nonce(authenticate)

						xml = ET.fromstring(r.text)
						
						eboks_attachment = xml
						
						# Gets if, name and format of attachment
						for child in eboks_attachment:
							for subtree in child:
								attachmentid = subtree.attrib['id']
								attachmenttitle = subtree.attrib['name']
								attachmentformat = subtree.attrib['format']
				
								# Gets the actual attachment
								url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + attachmentid + '/content'
									
								authstr = createauthstring()

								headers = {
								'X-EBOKS-AUTHENTICATE': authstr,
								'Accept': '*/*',
								'Accept-Language': 'en-US',
								'Host': data['root'],
								}

								r = requests.get(url, headers=headers)
								authenticate = r.headers['X-EBOKS-AUTHENTICATE']
								nonce(authenticate)
								
								# Attach attachment to e-mail
								if attachmentformat in ("txt","text","html","htm","plain"):
									filename = "".join().rstrip() + "." + attachmentformat
									r.encoding = "utf-8"
									part = MIMEText(r.text)
									part.add_header('Content-Disposition', 'attachment', filename = filename)
									msg.attach(part)
								elif attachmentformat == "pdf":
									filename = "".join().rstrip() + "." + attachmentformat
									part = MIMEApplication(r.content)
									part.add_header('Content-Disposition', 'attachment', filename = filename)
									msg.attach(part)
								elif attachmentformat in ("gif","jpg","jpeg","tiff","tif","webp"):
									filename = "".join().rstrip() + "." + attachmentformat
									part = MIMEImage(r.content)
									part.add_header('Content-Disposition', 'attachment', filename = filename)
									msg.attach(part)

					# Send e-mail
					if data['sendemails'] == True:
						print("sending")
						msg.attach(MIMEText(body, 'plain'))
						server.sendmail(data['emailfrom'], data['emailto'], msg.as_string())
						time.sleep(2)
Categories
blandet

Sådan trækker jeg links til gratis Zetland-artikler ud fra Zetlands Twitter-konto til wallnot.dk

På wallnot.dk udgiver jeg en liste over gratisartikler fra en lang række medier, der benytter sig af betalingsmure/paywall. Siden er ment som en service til brugere, der ved, at de gerne vil læse nyhedsartikler, og at de ikke vil betale for dem.

Zetland er ikke som de andre aviser. Der er ikke en forside med links til alle nypublicerede artikler. I stedet bruger Zetland Twitter til at lægge appetitvækkere ud.

Jeg syntes det var ærgerligt ikke at have Zetland med på Wallnot, så i stedet for at kigge efter links på forsiden, som Wallnot gør hos de andre medier, brugte jeg Twitters API til at trække artikellinks ud.

Her kan du se, hvordan jeg gjorde. Hvis du gerne vil prøve programmet af, skal du registrere dig som udvikler på Twitter.

# -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com
""" This program uses the Twitter API to get a list of free articles from Zetland """

import requests
from bs4 import BeautifulSoup
from datetime import datetime
from datetime import date
import json
from nested_lookup import nested_lookup
import base64


# GETS TWITTER DATA #

# Key and secret from Twitter developer account: https://developer.twitter.com/en/apply/user
client_key = ''
client_secret = ''

# Key and secret encoding, preparing for Twitter request
key_secret = '{}:{}'.format(client_key, client_secret).encode('ascii')
b64_encoded_key = base64.b64encode(key_secret)
b64_encoded_key = b64_encoded_key.decode('ascii')

base_url = 'https://api.twitter.com/'
auth_url = '{}oauth2/token'.format(base_url)

auth_headers = {
	'Authorization': 'Basic {}'.format(b64_encoded_key),
	'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'
}

auth_data = {
	'grant_type': 'client_credentials'
}

auth_resp = requests.post(auth_url, headers=auth_headers, data=auth_data)
auth_resp.json().keys()
access_token = auth_resp.json()['access_token']

search_headers = {
	'Authorization': 'Bearer {}'.format(access_token)
}

# Search parameters for Zetland tweets
search_params = {
	'user_id': 452898921,
	'count': 35,
	'tweet_mode': 'extended',
	'exclude_replies': 'true',
	'trim_user': 'true'
}

# Request url for searching user timelines
search_url = '{}1.1/statuses/user_timeline.json'.format(base_url)

# Request to Twitter
search_resp = requests.get(search_url, headers=search_headers, params=search_params)

# Response from Twitter in json format
tweet_data = search_resp.json()
#prettyjson = json.dumps(tweet_data, ensure_ascii=False, indent=4) # Only needed for debugging to pretify json

# Looks for all instances of expanded_url (that is, links) in json	
linklist = list(set(nested_lookup('expanded_url', tweet_data)))

# Populates a list of links to Zetland articles
urllist = []
for link in linklist:
	if "zetland.dk/historie" in link:
		urllist.append(link)

		
# GETS ARTICLE DATA FROM ZETLAND #
		
# Requests articles and get titles and dates and sort by dates directly from Zetland site
articlelist = []
titlecheck = []

for url in urllist:
	try:
		data = requests.get(url)
		result = data.text

		# Soup site and create a dictionary of links and their titles and dates
		articledict = {}
		soup = BeautifulSoup(result, "lxml")

		title = soup.find('meta', attrs={'property':'og:title'})
		title = title['content']
		
		timestamp = soup.find('meta', attrs={'property':'article:published_time'})
		timestamp = timestamp['content']
		timestamp = timestamp[:timestamp.find("+")]
		dateofarticle = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f')
		
		if title not in titlecheck:
			articlelist.append({"title": title, "url": url, "date": dateofarticle})
			titlecheck.append(title)
	except:
		print(url)


# PREPARES LIST OF ARTICLES FOR WALLNOT.DK #

# Sort articles by date (newest first)		
articlelist_sorted = sorted(articlelist, key=lambda k: k['date'], reverse=True) 

# Removes articles older than approximately three months
articlelist_recent = []
now = datetime.now()
for article in articlelist_sorted:
	timesincelast = now - article["date"]
	if timesincelast.days < 92:
		articlelist_recent.append(article)

# Converts dates to friendly format for display and outputs articles as html paragraphs
zet_linkstr = ""
for article in articlelist_recent:
	friendlydate = article["date"].strftime("%d/%m")
	zet_linkstr += '<p>' + friendlydate + ': ' + '<a href="' + article["url"] + '">' + article["title"] + '</a></p>\n' 

# Prints list of articles	
print(zet_linkstr)