Hvis du har været på internettet, er du sikkert en gang stødt på sådan ét her:
Mange webadministratorer vælger at skjule disse oversigter over filer på en webserver, som webserversoftwaren Apache kan generere automatisk.
Men jeg opdagede ved et tilfælde, at jeg kunne se, hvad fotoagenturet Magnum havde lagt op i deres WordPress-installation.
Jeg besluttede at forsøge at lave en lokal kopi, så jeg kunne kigge på flotte fotografier uden at skulle vente på downloads fra internettet.
Først forsøgte jeg med Wget, som er et lille program, der er designet til at dublere websteder lokalt. Men Wget havde problemer med at hente og tygge sig igennem de lange lister med filer. En af dem fyldte fx 36 megabytes. Det er altså rigtig mange links.
Derfor lavede jeg et lille Python-program, der kan tygge sig igennem denne type mappe- og filoversigter og downloade dem lokalt.
Her er det:
# apache-directory-downloader.py
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com
'''A program to fetch files from standard apache directory listings on the internet.
See https://duckduckgo.com/?t=ffab&q=apache%2Bdirectory%2Blisting&ia=images&iax=images
for examples of what this is.'''
import requests # Send http requests and receive responses
from bs4 import BeautifulSoup # Parse HTML data structures, e.g. to search for links
import os # Used to create directories at local destination
import shutil # Used to copy binary files from http response to local destination
import re # Regex parser and search functions
# Terms to exclude, files with these strings in them are not downloaded
exclude = [
"-medium",
"-overlay",
"-teaser-",
"-overlay",
"-thumbnail",
"-collaboration",
"-scaled",
"-photographer-featured",
"-photographer-listing",
"-full-on-mobile",
"-theme-small-teaser",
"-post",
"-large",
"-breaker",
]
# Takes an url and collects all links
def request(url, save_location):
# Print status to let user know that something is going on
print("Requesting:", url)
# Fetch url
response = requests.get(url)
# Parse response
soup = BeautifulSoup(response.text, "lxml")
# Search for all links and exclude certain strings and patterns from links
urllist = [a['href'] for a in soup.find_all('a', href=True) if not '?C=' in a['href'] and not a['href'][0] == "/" and not any(term in a['href'] for term in exclude) and not re.search("\d\d[x]\d\d",a['href'])]
# If status code is not 200 (OK), add url to list of errors
if not response.status_code == 200:
errorlist.append(url)
# Send current url, list of links and current local save collection to scrape function
return scrape(url, urllist, save_location)
def scrape(path, content, save_location):
# Loop through all links
for url in content:
# Print status to let user know that something is going on
print("Parsing/downloading:", path+url)
# If there's a slash ("/") in the link, it is a directory
if "/" in url:
# Create local directory if it doesn't exists
try:
os.mkdir(save_location+url)
except:
pass
# Run request function to fetch contents of directory
request(path+url, save_location+url)
# If the link doesn't contain a slash, it's a file and is saved
else:
# Check if file already exists, e.g. has been downloaded in a prior run
if not os.path.isfile(save_location+url):
# If file doesn't exist, fetch it from remote location
file = requests.get(path+url, stream=True)
# Print status to let user know that something is going on
print("Saving file:", save_location+url)
# Save file to local destination
with open(save_location+url, 'wb') as f:
# Decodes file if received compressed from server
file.raw.decode_content = True
# Copies binary file to local destination
shutil.copyfileobj(file.raw, f)
# List to collect crawling errors
errorlist = []
# Local destination, e.g. 'C:\Downloads' for Windows
save_location = "C:/Downloads/"
# Remote location, e.g. https://example.com/files
url = "https://content.magnumphotos.com/wp-content/uploads/"
# Call function to start crawling
request(url, save_location)
# Print any crawling errors
print(errorlist)
Til Wallnot ville jeg gerne have fat i samtlige Zetland-historier, som Google har indekseret.
Til det formål skrev jeg et lille program, der gennemgår Googles søgeresultater. Programmet holder en lille pause mellem hver side med søgeresultater, der hentes. Det skyldes at Google åbenbart ikke selv er vild med robotter, paradoksalt nok.
import requests
from bs4 import BeautifulSoup
import time
import random
linkcollection = []
def google_results(url):
try:
result = requests.get(url)
soup = BeautifulSoup(result.text, "lxml")
links = soup.find_all('a')
for link in links:
if "zetland.dk/historie/" in link['href']:
full_link = link['href']
url = full_link[full_link.find("q=")+2:full_link.find("&")]
linkcollection.append(link['href'])
print(link['href'])
next_page = soup.find('a', attrs={'aria-label': 'Næste side'})
time_to_sleep = random.randrange(3,7)
print("Sleeping " + str(time_to_sleep) + " seconds")
time.sleep(time_to_sleep)
google_results('https://www.google.com'+next_page['href'])
except TypeError:
print("No more results it seems")
url = 'https://www.google.com/search?q=site:zetland.dk/historie'
google_results(url)
with open("./googlelist.txt", "wt", encoding="utf8") as fout:
fout.write(str(linkcollection))
Der er nok ikke mange mennesker efterhånden, der ikke har deres egen kortlinkservice. En af de mest kendte er https://bitly.com/.
Som en øvelse har jeg lavet kortlinkservicen https://wallnot.dk/link. Linkene bliver godt nok ikke specielt korte, men indtil videre sparer jeg udgiften til et selvstændigt domænenavn. Det er ikke fordi, der mangler muligheder andre steder.
At lave et kortlink-værktøj i Django er overraskende nemt.
Jeg starter med min datamodel i models.py. Hvert link har en destination (det lange link), et kort link og et tidsstempel. Destinationen er en URL, det korte link er et antal tegn og tidsstemplet er – et tidsstempel:
from django.db import models
from django.utils import timezone
class Link(models.Model):
destination = models.URLField(max_length=500)
shortlink = models.CharField(max_length=6, unique=True)
date = models.DateTimeField(default=timezone.now, editable=False)
Jeg ved, at jeg skal bruge en formular. Den opretter jeg i forms.py. Her bruger jeg en type formular, der kaldes ModelForm. Django sørger for, at valideringsreglerne følger samme type data, som jeg har i min bagvedliggende datamodel:
from django.forms import ModelForm, URLInput
from .models import Link
class LinkForm(ModelForm):
class Meta:
model = Link
fields = ['destination']
widgets = {
'destination': URLInput(attrs={'placeholder': 'Indsæt link'}),
}
Logikkerne bag de enkelte visninger i Django laves i views.py. Jeg har to forskellige visninger. Én visning som jeg bruger til at vise min forside, hvor jeg både viser min formular til indtastning af links og det korte link (index). Én visning, som aktiveres når brugeren besøger et kort link (redirect).
Endelig har jeg en funktion, som jeg bruger til at generere selve de korte links.
Jeg har kommenteret koden en masse, så jeg håber den er til at følge med i:
from django.shortcuts import render
from django.http import HttpRequest, HttpResponseRedirect
from .models import Link
from .forms import LinkForm
import hashlib
import bcrypt
# Function to create a random hash to use as short link address
def create_shortlink(destination):
salt = bcrypt.gensalt().decode() # Random salt
destination = destination+salt # Salt added to destination URL
hash = hashlib.md5(destination.encode()).hexdigest() # Hashed to alphanumeric string
return hash[:6] # First 6 characters of that string
# Front page with a form to enter destination address. Short URL returned.
def index(request):
form = LinkForm() # Loads form
url = 'https://wallnot.dk/link/' # site url
# If a destination is submitted, a short link is returned
if request.method == 'POST':
form = LinkForm(request.POST) # Form instance with submitted data
# Check whether submitted data is valid
if form.is_valid():
destination = form.cleaned_data['destination'] # Submitted destination
# If destination is already in database, return short link for destination from database
try:
link = Link.objects.get(destination=destination)
sharelink = url + link.shortlink # Creates full URL using page URL and hash
# If destination is not in database, create a new short link
except:
# Loop to create a unique hash value for short link
unique_link = False
while unique_link == False:
hash = create_shortlink(destination) # Return hash
# First we check whether the hash is a duplicate
try:
Link.objects.get(shortlink=hash) # Check whether hash is used
# If not a duplicate, an error is thrown, and we can save the hash
except:
link = form.save(commit=False) # Prepare to save form destination data and hash
link.shortlink = hash # Sets short link to hash value
link.save() # Saves destination and short link to database
sharelink = url + link.shortlink # Creates full URL using page URL and hash
unique_link = True # If check causes error, hash is unused, exit loop
context = {'sharelink': sharelink, 'form': form} # Dictionary with variables used in template
return render(request, 'links/index.html', context)
# If form is invalid, just renders page.
else:
context = {'form': form}
return render(request, 'links/index.html', context)
# Render page with form before user has submitted
context = {'form': form}
return render(request, 'links/index.html', context)
# Short link redirect to destination URL
def redirect(request, shortlink):
# Query the database for short link, if there is a hit, redirect to destination URL
try:
link = Link.objects.get(shortlink=shortlink)
return HttpResponseRedirect(link.destination)
# An error means the short link doesn't exist, so the front page template is shown with an error variable
except:
error = True
context = {'error': error}
return render(request, 'links/index.html', context)
For at kunne servere siderne, har jeg urls.py, der fortæller Django hvordan en indtastet URL af brugeren skal pege på funktioner i views.py:
from django.urls import path
from . import views
urlpatterns = [
path('', views.index, name='index'),
path('<shortlink>', views.redirect, name='redirect'),
]
Og endelig har jeg index.html, som er den skabelon, som min side genereres på baggrund af. Hvis du ikke har prøvet Django før, så læg mærke til alt det, der står i tuborgklammer ({}). De bruges dels til simple funktioner (fx if-funktioner, dels til at indsætte variable fra views.py i den side, der genereres.
Hvis du lægger mærke til funktionerne, bruger jeg if-funktionerne til at nøjes med en skabelon, uanset hvilken situation brugeren er havnet i, sådan at indholdet fx er anderledes, når brugeren har lavet en fejl i udfyldelsen af formularen, end når brugeren ikke har udfyldt formularen endnu.
Der er også et lille javascript i filen, der sørger for at brugeren kan kopiere det korte link til sin udklipsholder.
<!doctype html>
<html lang="da">
<head>
<!-- Required meta tags -->
<title>Korte links</title>
<meta name="description" content="Skønne korte links">
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
<link rel="manifest" href="/site.webmanifest">
<link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5">
<meta name="msapplication-TileColor" content="#ffc40d">
<meta name="theme-color" content="#ffffff">
<style>
body {
font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";
text-align: center;
box-sizing: border-box;
}
h1 {
margin-top: 0;
font-size: 4.0rem;
font-weight: 300;
line-height: 1.2;
margin-bottom: 1.5rem;
}
h2 {
margin-top: 1.5rem;
font-size: 2.5rem;
font-weight: 300;
line-height: 1.2;
margin-bottom: 1.5rem;
}
input {
width: 60%;
line-height: 1.2;
font-size: 1.0rem;
height: 1.5rem;
padding: 10px;
}
button {
width: 50%;
border: 1px solid transparent;
padding: .375rem .75rem;
font-size: 1rem;
line-height: 1.8;
height: 2.5rem;
border-radius: .25rem;
color: #fff;
background-color: #28a745;
border-color: #28a745;
}
button:focus {
box-shadow: 0 0 0 0.2rem rgba(72,180,97,.5)
}
button:hover {
background-color: #218838;
border-color: #1e7e34;
}
.footer {
position: fixed;
left: 0;
bottom: 0;
width: 100%;
background-color: #f1f1f1;
color: black;
}
</style>
</head>
<body>
<h1>Lav et kort link</h1>
{% if form %}
<form method="post">
{% csrf_token %}
<p>{{ form.destination }}</p>
<p><button type="submit" value="Lav et kort link">Lav et kort link</button></p>
</form>
{% if form.destination.errors %}
<h2>Tast et gyldigt link!</h2>
<p><em>Du har tastet et ugyldigt link. Prøv igen med et gyldigt link med http://, https://, ftp:// eller ftps:// foran.</em></p>
{% endif %}
{% if request.method == "POST" and not form.destination.errors %}
<h2>Her er dit link:</h2>
<p><a href="{{ sharelink }}">{{ sharelink }}</a></p>
<button class="copy">Kopier link</button>
{% endif %}
{% endif %}
{% if error %}
<h2>Har du tastet forkert?</h2>
<p><em>Du har prøvet at bruge et kort link. Desværre er det link, du har tastet, ikke registreret. Måske er du kommet til at taste forkert?</em></p>
<p><a href="{% url 'index' %}">Til forsiden</a>
{% endif %}
<div class="footer">
<p>Lav relativt korte links på wallnot.dk. Gratis og fri for annoncer og overvågning.</p>
</div>
<script>
function fallbackCopyTextToClipboard(text) {
var textArea = document.createElement("textarea");
textArea.value = text;
document.body.appendChild(textArea);
textArea.focus();
textArea.select();
try {
var successful = document.execCommand("copy");
var msg = successful ? "successful" : "unsuccessful";
console.log("Fallback: Kopiering gik fint " + msg);
} catch (err) {
console.error("Fallback: Kunne ikke kopiere", err);
}
document.body.removeChild(textArea);
}
function copyTextToClipboard(text) {
if (!navigator.clipboard) {
fallbackCopyTextToClipboard(text);
return;
}
navigator.clipboard.writeText(text).then(function() {
console.log('Kopiering gik fint');
}, function(err) {
console.error('Kunne ikke kopiere', err);
});
}
var copy = document.querySelector('.copy');
copy.addEventListener('click', function(event) {
copyTextToClipboard('{{ sharelink }}');
});
</script>
</body>
</html>
For tiden øver jeg mig i at bruge Django – et værktøj til at lave webapplikationer i Python. Det er vildt smart.
Det tog et par timer at få https://wallnot.dk/pak/ i luften, men så er der heller ikke gjort noget ud af brugerfladen og det bagvedliggende kunne helt sikkert også gøres smartere. Siden kan bruges til at spore pakker til levering fra flere forskellige transportører (PostNord, GLS, DAO).
Hvis du har pakker på vej fra andre transportører og vil dele pakkenumrene med mig, er jeg interesseret.
Min Python-simulation af kortspillet Krig var ikke særlig elegant. Ved krig og dobbelt-krig osv. var en masse “if”-sætninger inde i hinanden med samme logik. (Jeg fandt også nogle dumme fejl, så jeg har opdateret det oprindelige indlæg.)
Derfor har jeg prøvet at skrive en ny version.
Den fungerer fint og giver følgende output ved 1.000.000 spil:
Der blev spillet 1000000 spil
Det gennemsnitlige antal dueller var 177.217668
Det højeste antal dueller var 2238
Det laveste antal dueller var 3
Den spiller med højest sum af kort vandt 573276 gange (57%)
Den spiller med højest sum af kort tabte 397771 gange (40%)
Uafgjorte spil: 1
Antal enkeltkrig, dobbeltkrig, osv.: 12348559, 886651, 60655, 3722, 218, 11, 2
Vendte kort uden krig og med krig: 176766958, 13299818
Spillene tog 225.4 sekunder
Det nye program:
# KRIG #
import time
start_time = time.time()
import random
number_of_games_to_play = 1000000
number_of_games_counter = 0
number_of_plays_list = []
highest_deck_won = 0
highest_deck_lost = 0
equal_games = 0
war_types = [0,0,0,0,0,0,0]
war_or_not_war = [0,0]
# Loop to play games
percentage_copy = 0
i = 0
while i < number_of_games_to_play:
# One is added to i so loop finishes once number of games have been played
i += 1
# Prints percentage done with 1 decimal every time it changes
percentage_completed = round((i/number_of_games_to_play*100), 1)
if percentage_copy != percentage_completed:
print("{}% done".format(percentage_completed))
percentage_copy = percentage_completed
# Create a deck, shuffle it and divide between players
deck = [2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11,12,12,12,12,13,13,13,13,14,14,14,14]
random.shuffle(deck)
player_a_deck = deck[0:26]
player_b_deck = deck[26:52]
# Which player has the highest sum of cards
card_sum_a = sum(player_a_deck)
card_sum_b = sum(player_b_deck)
if card_sum_a > card_sum_b:
highest_deck = "a"
elif card_sum_a < card_sum_b:
highest_deck = "b"
else:
highest_deck = "equal"
# Loop to turn cards within games
number_of_plays = 0
index = 1
while True:
try:
if index == 1:
number_of_plays += 1 # Add 1 to number of plays counter
war_count = 0 # Reset war counter
# Player a has the largest card
if player_a_deck[index-1] > player_b_deck[index-1]:
war_or_not_war[0] += 1
player_a_deck.extend(player_a_deck[:index])
player_a_deck.extend(player_b_deck[:index])
del player_a_deck[:index]
del player_b_deck[:index]
index = 1 # If a play is decided, index is reset
# Player b has the largest card
elif player_a_deck[index-1] < player_b_deck[index-1]:
war_or_not_war[0] += 1
# Cards are added in different order to deck in order to avoid (game) risk of going on forever (infinite loop)!
player_b_deck.extend(player_b_deck[:index])
player_b_deck.extend(player_a_deck[:index])
del player_a_deck[:index]
del player_b_deck[:index]
index = 1 # If a play is decided, index is reset
# War is on!
else:
war_or_not_war[1] += 1
index += 4 # In case of war the index is upped by four cards
war_types[war_count] += 1
war_count += 1
# If a player has too few cards left to participate, game is over
except IndexError:
# If a player had no cards left and index is 1, the game was already over, so number of plays is corrected
if index == 1:
number_of_plays -= 1
break
# Single game is over #
# Compare deck sizes to decide winner and add values to counters and lists
deck_a = len(player_a_deck)
deck_b = len(player_b_deck)
if deck_a > deck_b:
if highest_deck == "a":
highest_deck_won += 1
elif highest_deck == "b":
highest_deck_lost += 1
elif deck_a < deck_b:
if highest_deck == "a":
highest_deck_lost += 1
elif highest_deck == "b" :
highest_deck_won += 1
else:
equal_games += 1
number_of_plays_list.append(number_of_plays)
number_of_games_counter += 1
# All games are over #
print("Der blev spillet {} spil".format(number_of_games_counter))
print("Det gennemsnitlige antal dueller var {}".format(sum(number_of_plays_list)/len(number_of_plays_list)))
print("Det højeste antal dueller var {}".format(max(number_of_plays_list)))
print("Det laveste antal dueller var {}".format(min(number_of_plays_list)))
print("Den spiller med højest sum af kort vandt {} gange ({}%)".format(highest_deck_won, round(highest_deck_won/number_of_games_counter*100)))
print("Den spiller med højest sum af kort tabte {} gange ({}%)".format(highest_deck_lost, round(highest_deck_lost/number_of_games_counter*100)))
print("Uafgjorte spil: {}".format(equal_games))
print("Antal enkeltkrig, dobbeltkrig, osv.: {}".format(", ".join(str(x) for x in war_types)))
print("Vendte kort uden krig og med krig: {}".format(", ".join(str(x) for x in war_or_not_war)))
print("Spillene tog {} sekunder".format(round(time.time() - start_time, 1)))
Hvis man tilfældigvis har et barn i 5-årsalderen, kan man spille kortspillet Krig. Kortene blandes og deles ligeligt mellem 2 spillere, hver spiller vender et kort fra sin bunke samtidig, højeste kort vender, hvis kortene er lige høje, er der krig. Det er så enkelt, at man lige så godt kunne få en computer til at spille det.
Derfor skrev jeg et lille program i Python, der kan simulere kortspillet.
Jeg opdagede et hul i reglerne: Der er ingen steder, der beskriver, hvad der sker, når en spiller ikke har kort nok til at deltage i en krig (eller en dobbelt-krig, tredobbelt-krig, osv.) Jeg besluttede, at hvis en spiller på et tidspunkt mangler kort til at kunne deltage, taber den spiller, der ikke har kort nok til at deltage. I den meget sjældne situation, at begge spillere ikke har nok kort til at deltage (en mange-mange-dobbelt-krig i starten af spillet), vinder den spiller, der har flest kort. Har begge spillere lige mange kort, bliver det uafgjort.
Jeg fik computeren til at spille 1 million spil Krig, og her er hvad jeg kan fortælle dig om Krig, som du ikke vil vide:
Det gennemsnitlige antal dueller i et spil krig er 177
Spillet med flest dueller havde 1.825 dueller
Spillet med færrest havde 4 dueller
Spilleren med den højeste sum af kort efter kortene blev blandet vandt 573.405 gange
Spilleren med den laveste sum af kort vandt 397.602 gange
I løbet af spillene blev der spillet:
Enkeltkrig: 12.366.762 gange
Dobbeltkrig: 888.024 gange
Trippelkrig: 60.727 gange
Firdobbeltkrig: 3.852 gange
Femdobbeltkrig: 206 gange
Seksdobbeltkrig: 10 gange
Her er programmet:
import random
krig1 = 0
krig2 = 0
krig3 = 0
krig4 = 0
krig5 = 0
krig6 = 0
krig7 = 0
number_of_plays_list = []
not_war = 0
war = 0
highest_deck_won = 0
highest_deck_lost = 0
equal_games = 0
i = 0
number_of_games = 1000000
while i <= number_of_games:
number_of_plays_counter = 0
deck = [2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11,12,12,12,12,13,13,13,13,14,14,14,14]
random.shuffle(deck)
player_a_deck = deck[0:26]
player_b_deck = deck[26:52]
if sum(player_a_deck) > sum(player_b_deck):
highest_deck = "a"
elif sum(player_a_deck) < sum(player_b_deck):
highest_deck = "b"
else:
highest_deck = "equal"
while len(player_a_deck) > 0 and len(player_b_deck) > 0:
number_of_plays_counter += 1
if player_a_deck[0] > player_b_deck[0]:
not_war += 1
player_a_deck.append(player_a_deck[0])
player_a_deck.append(player_b_deck[0])
del player_a_deck[0]
del player_b_deck[0]
elif player_a_deck[0] < player_b_deck[0]:
not_war += 1
player_b_deck.append(player_b_deck[0])
player_b_deck.append(player_a_deck[0])
del player_a_deck[0]
del player_b_deck[0]
elif player_a_deck[0] == player_b_deck[0]:
war += 1
krig1 += 1
if len(player_a_deck) >= 5 and len(player_b_deck) >= 5:
if player_a_deck[4] > player_b_deck[4]:
player_a_deck.extend(player_a_deck[0:5])
player_a_deck.extend(player_b_deck[0:5])
del player_a_deck[0:5]
del player_b_deck[0:5]
elif player_a_deck[4] < player_b_deck[4]:
player_b_deck.extend(player_b_deck[0:5])
player_b_deck.extend(player_a_deck[0:5])
del player_a_deck[0:5]
del player_b_deck[0:5]
elif player_a_deck[4] == player_b_deck[4]:
krig2 += 1
if len(player_a_deck) >= 9 and len(player_b_deck) >= 9:
if player_a_deck[8] > player_b_deck[8]:
player_a_deck.extend(player_a_deck[0:9])
player_a_deck.extend(player_b_deck[0:9])
del player_a_deck[0:9]
del player_b_deck[0:9]
elif player_a_deck[8] < player_b_deck[8]:
player_b_deck.extend(player_b_deck[0:9])
player_b_deck.extend(player_a_deck[0:9])
del player_a_deck[0:9]
del player_b_deck[0:9]
elif player_a_deck[8] == player_b_deck[8]:
krig3 += 1
if len(player_a_deck) >= 13 and len(player_b_deck) >= 13:
if player_a_deck[12] > player_b_deck[12]:
player_a_deck.extend(player_a_deck[0:13])
player_a_deck.extend(player_b_deck[0:13])
del player_a_deck[0:13]
del player_b_deck[0:13]
elif player_a_deck[12] < player_b_deck[12]:
player_b_deck.extend(player_b_deck[0:13])
player_b_deck.extend(player_a_deck[0:13])
del player_a_deck[0:13]
del player_b_deck[0:13]
elif player_a_deck[12] == player_b_deck[12]:
krig4 += 1
if len(player_a_deck) >= 17 and len(player_b_deck) >= 17:
if player_a_deck[16] > player_b_deck[16]:
player_a_deck.extend(player_a_deck[0:17])
player_a_deck.extend(player_b_deck[0:17])
del player_a_deck[0:17]
del player_b_deck[0:17]
elif player_a_deck[16] < player_b_deck[16]:
player_b_deck.extend(player_b_deck[0:17])
player_b_deck.extend(player_a_deck[0:17])
del player_a_deck[0:17]
del player_b_deck[0:17]
elif player_a_deck[16] == player_b_deck[16]:
krig5 += 1
if len(player_a_deck) >= 21 and len(player_b_deck) >= 21:
if player_a_deck[20] > player_b_deck[20]:
player_a_deck.extend(player_a_deck[0:21])
player_a_deck.extend(player_b_deck[0:21])
del player_a_deck[0:21]
del player_b_deck[0:21]
elif player_a_deck[20] < player_b_deck[20]:
player_b_deck.extend(player_b_deck[0:21])
player_b_deck.extend(player_a_deck[0:21])
del player_a_deck[0:21]
del player_b_deck[0:21]
elif player_a_deck[20] == player_b_deck[20]:
krig6 += 1
if len(player_a_deck) >= 25 and len(player_b_deck) >= 25:
if player_a_deck[24] > player_b_deck[24]:
player_a_deck.extend(player_a_deck[0:25])
player_a_deck.extend(player_b_deck[0:25])
del player_a_deck[0:25]
del player_b_deck[0:25]
elif player_a_deck[24] < player_b_deck[24]:
player_b_deck.extend(player_b_deck[0:25])
player_b_deck.extend(player_a_deck[0:25])
del player_a_deck[0:25]
del player_b_deck[0:25]
elif player_a_deck[24] == player_b_deck[24]:
krig7 += 1
break
else:
break
else:
break
else:
break
else:
break
else:
break
else:
break
if len(player_a_deck) > len(player_b_deck):
if highest_deck == "a":
highest_deck_won += 1
elif highest_deck == "b" :
highest_deck_lost += 1
elif len(player_a_deck) < len(player_b_deck):
if highest_deck == "a":
highest_deck_lost += 1
elif highest_deck == "b" :
highest_deck_won += 1
else:
equal_games += 1
number_of_plays_list.append(number_of_plays_counter)
i += 1
print(i/number_of_games)
print("Der blev spillet {} spil".format(number_of_games))
print("Det gennemsnitlige antal dueller var {}".format(sum(number_of_plays_list)/len(number_of_plays_list)))
print("Det højeste antal dueller var {}".format(max(number_of_plays_list)))
print("Det laveste antal dueller var {}".format(min(number_of_plays_list)))
print("Den spiller med højest sum af kort vandt {} gange".format(highest_deck_won))
print("Den spiller med højest sum af kort tabte {} gange".format(highest_deck_lost))
print(krig1, krig2, krig3, krig4, krig5, krig6, krig7)
print(not_war, war)
print("Uafgjorte spil: {}".format(equal_games))
Opdateret d. 20/10/2019: Nogle gange har Saxo Bank en “disclaimer” (i dette tilfælde omkring Brexit), som de vil vise, inden man får lov at logge på. Jeg har tilpasset koden, sådan programmet kan håndtere dette tilfælde.
Nu er jeg også blevet kunde hos Saxo Bank. (Hvorfor? Mulighed for at oprette en aktiesparekonto og ingen minimumskurtage.)
Selvbetjeningsløsningen hos Saxo Bank er rigtig dårlig (sammenlignet med Nordnets).
Derfor var jeg interesseret i, om jeg kunne finde en måde at trække transaktionsdata ud fra min konto hos Saxo Bank – uden at have brug for at se på hjemmesiden.
Allerførst kiggede jeg på, hvad hjemmesiden gør, når den skal vise mine data. Jeg valgte gennemførte handler i menuen:
Og så kiggede jeg på, hvad browseren gjorde. Det viser sig at hjemmesiden – fornuftigt nok – bruger Saxo Banks API, når den skal vise data til brugeren:
Jeg kunne se, at API’et modtog en lang streng (“Authorization” med ordet BEARER foran). Den gik jeg ud fra, var nødvendig, for at få data tilbage.
Så spørgsmålet var egentlig bare: Hvordan bliver sådan en BEARER-streng genereret?
Tilbage til start
For at komme frem til, hvordan BEARER-strengen genereres, gik jeg tilbage til start: Jeg gik til loginsiden og trykkede F12 i min browser (Chrome) for at følge med i netværksforespørgslerne.
Loginsidens formular sender mit brugernavn og password af sted, sammen med en streng – “AuthnRequest” – der genereres på ny hver gang, loginsiden hentes:
I mit Python-program prøver jeg at sende sådan en formular af sted, og undersøger hvad jeg får tilbage.
# Visit login page and get AuthnRequest token value from input form
url = 'https://www.saxoinvestor.dk/Login/da/'
r = requests.get(url)
soup = BeautifulSoup(r.text, "html.parser")
input = soup.find_all('input', {"id":"AuthnRequest"})
authnrequest = input[0]["value"]
# Login step 1: Submit username, password and token and get another token back
url = 'https://www.saxoinvestor.dk/Login/da/'
r = requests.post(url, data = {'field_userid': user, 'field_password': password, 'AuthnRequest': authnrequest})
Lidt forkortet ser det sådan her ud:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta name="Application-State" content="service=IDP;federated=False;env=Live;state=Ok;authenticated=True;"><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
</head>
<body>
<noscript><p><strong>Note:</strong> Since your browser does not support Javascript, you must press the Continue button to proceed.</p></noscript>
<form id="form" action="https://www.saxoinvestor.dk/investor/login.sso.ashx" method="post"><div>
<input type="hidden" name="SAMLResponse" value="PHNhbWxwOlJlc3BvbnNlIElEPSJfNjQzZGI4ODQtMDMzZi00MWVhLWE4ZjEtYzVjOWVlMWIxM2IwIiBJblJlc3BvbnNlVG89Il9mN2E3ODBlZi0yZjdmLTQyYmItODk1[...]G9uc2U+"/>
<input type="hidden" name="RelayState" value=""/>
<input type="hidden" name="PageLoadInfo" id="PageLoadInfo" value=""/></div>
<noscript><div>
<input type="submit" value="Continue"/></div></noscript></form><script type="text/javascript">function doSubmit(){var t=-1;if(window.location.hash){var m=window.location.hash.match(/\/lst\/(\d+)/);if(m) t=parseInt(m[1]);}if(t>=0 && document.getElementById("PageLoadInfo").value=='')document.getElementById("PageLoadInfo").value=t;document.forms.form.submit();}</script><script type="text/javascript">doSubmit();</script></body></html>
Og hvad er det så? En formular (“<input>”) til browsere uden Javascript, med et felt der hedder “SAMLResponse” med en lang (her forkortet) streng som værdi.
I Chrome kan jeg se, at allersidste trin. inden jeg når ind på forsiden af selvbetjeningen, faktisk er, at min browser sender “SAMLResponse” af sted til en side, der hedder “login.sso.ashx”:
Så jeg sender trygt formularen af sted med SAMLResponse-.værdien og ser, hvad jeg får tilbage:
soup = BeautifulSoup(r.text, "html.parser")
input = soup.find_all('input', {"name":"SAMLResponse"})
samlresponse = input[0]["value"]
# Login step 2: Get bearer token necessary for API requests
url = 'https://www.saxoinvestor.dk/investor/login.sso.ashx'
r = requests.post(url, data = {'SAMLResponse': samlresponse})
Og vupti: Siden videresender mig til API’et med et BEARER-token, jeg kan benytte mig af. Det får jeg fat i (og skærer lidt til) sådan her::
Og så er jeg ellers klar til at hente data fra API’et. Det starter sådan her:
# START API CALLS
# Documentation at https://www.developer.saxo/openapi/learn
# Set bearer token as header
headers = {'Authorization': bearer}
# First API request gets Client Key which is used for most API calls
# See https://www.developer.saxo/openapi/learn/the-tutorial for expected return data
url = 'https://www.saxoinvestor.dk/openapi/port/v1/clients/me'
r = requests.get(url, headers=headers)
clientdata = r.json()
clientkey = clientdata['ClientKey']
Hele programmet
Hele programmet – inklusive den måde, jeg omdanner Saxo Bank-data til samme format som Nordnets transaktionsdata – er her.
Du må gøre præcis som du har lyst til med det (på eget ansvar).
# -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com
"""This program logs into a Saxo Bank account and lets you make API requests."""
import requests
from datetime import datetime
from datetime import date
from bs4 import BeautifulSoup
import json
# USER ACCOUNT AND PERIOD DATA. SHOULD BE EDITED FOR YOUR NEEDS #
# Saxo user account credentials
user = '' # your user id
password = '' # your password
# Start date (start of period for transactions) and date today used for extraction of transactions
startdate = '2019-01-01'
today = date.today()
enddate = datetime.strftime(today, '%Y-%m-%d')
# LOGIN TO SAXO BANK
# Visit login page and get AuthnRequest token value from input form
url = 'https://www.saxoinvestor.dk/Login/da/'
r = requests.get(url)
soup = BeautifulSoup(r.text, "html.parser")
input = soup.find_all('input', {"id":"AuthnRequest"})
authnrequest = input[0]["value"]
# Login step 1: Submit username, password and token and get another token back
url = 'https://www.saxoinvestor.dk/Login/da/'
r = requests.post(url, data = {'field_userid': user, 'field_password': password, 'AuthnRequest': authnrequest})
soup = BeautifulSoup(r.text, "html.parser")
input = soup.find_all('input', {"name":"SAMLResponse"})
# Most of the time this works
if input:
samlresponse = input[0]["value"]
# But sometimes there's a disclaimer that Saxo Bank would like you to accept
else:
input = soup.find_all('input')
inputs = {}
try:
for i in input:
inputs[i['name']] = i['value']
except:
pass
url = 'https://www.saxotrader.com/disclaimer'
request = requests.post(url, data=inputs)
cook = request.cookies['DisclaimerApp']
returnurl = cook[cook.find("ReturnUrl")+10:cook.find("&IsClientStation")]
url = 'https://live.logonvalidation.net/complete-app-consent/' + returnurl[returnurl.find("complete-app-consent/")+21:]
request = requests.get(url)
soup = BeautifulSoup(request.text, "html.parser")
input = soup.find_all('input', {"name":"SAMLResponse"})
samlresponse = input[0]["value"]
# Login step 2: Get bearer token necessary for API requests
url = 'https://www.saxoinvestor.dk/investor/login.sso.ashx'
r = requests.post(url, data = {'SAMLResponse': samlresponse})
bearer = r.history[0].headers['Location']
bearer = bearer[bearer.find("BEARER"):bearer.find("/exp/")]
bearer = bearer.replace("%20"," ")
# START API CALLS
# Documentation at https://www.developer.saxo/openapi/learn
# Set bearer token as header
headers = {'Authorization': bearer}
# First API request gets Client Key which is used for most API calls
# See https://www.developer.saxo/openapi/learn/the-tutorial for expected return data
url = 'https://www.saxoinvestor.dk/openapi/port/v1/clients/me'
r = requests.get(url, headers=headers)
clientdata = r.json()
clientkey = clientdata['ClientKey']
# Example API call #1
url = 'https://www.saxoinvestor.dk/openapi/cs/v1/reports/aggregatedAmounts/' + clientkey + '/' + startdate + '/' + enddate + '/'
r = requests.get(url, headers=headers)
data = r.json()
# Working on that data to add some transaction types to personal system
saxoaccountname = "Aktiesparekonto: Saxo Bank"
currency = "DKK"
saxotransactions = ""
for item in data['Data']:
if item['AffectsBalance'] == True:
date = item['Date']
amount = item['Amount']
amount_str = str(amount).replace(".",",")
if item['UnderlyingInstrumentDescription'] == 'Cash deposit or withdrawal' or item['UnderlyingInstrumentDescription'] == 'Cash inter-account transfer':
if amount > 0:
transactiontype = 'INDBETALING'
elif amount < 0:
transactiontype = 'HÆVNING'
saxotransactions += ";" + date + ";" + date + ";" + date + ";" + transactiontype + ";;;;;;;;" + amount_str + ";" + currency + ";;;;;;;;;" + saxoaccountname + "\r\n"
if item['AmountTypeName'] == 'Corporate Actions - Cash Dividends':
transactiontype = "UDB."
if item['InstrumentDescription'] == "Novo Nordisk B A/S":
paper = "Novo B"
papertype = "Aktie"
if item['InstrumentDescription'] == "Tryg A/S":
paper = "TRYG"
papertype = "Aktie"
saxotransactions += ";" + date + ";" + date + ";" + date + ";" + transactiontype + ";" + paper + ";" + papertype + ";;;;;;" + amount_str + ";" + currency + ";;;;;;;;;" + saxoaccountname + "\n"
# Example API call #2
url = "https://www.saxoinvestor.dk/openapi/cs/v1/reports/trades/" + clientkey + "?fromDate=" + startdate + "&" + "toDate=" + enddate
r = requests.get(url, headers=headers)
data = r.json()
# Working on that data to add trades to personal system
for item in data['Data']:
date = item['AdjustedTradeDate']
numberofpapers = str(int(item['Amount']))
amount_str = str(item['BookedAmountAccountCurrency']).replace(".",",")
priceperpaper = str(item['BookedAmountAccountCurrency'] / item['Amount']).replace(".",",")
if item['TradeEventType'] == 'Bought':
transactiontype = "KØBT"
if item['AssetType'] == 'Stock':
papertype = "Aktie"
if item['InstrumentDescription'] == "Novo Nordisk B A/S":
paper = "Novo B"
isin = "DK0060534915"
if item['InstrumentDescription'] == "Tryg A/S":
paper = "TRYG"
isin = "DK0060636678"
saxotransactions += ";" + date + ";" + date + ";" + date + ";" + transactiontype + ";" + paper + ";" + papertype + ";" + isin + ";" + numberofpapers + ";" + priceperpaper + ";;;" + amount_str + ";" + currency + ";;;;;;;;;" + saxoaccountname + "\n"
OPDATERING 29. maj 2019: Dette program virker ikke længere. Prøv dette i stedet.
Der er mange, der har fået den idé, automatisk at videresende beskeder i E-boks til en anden mailadresse eller gemme dem på computeren. Videresendelse kan også gøres i E-boks selv, men kun manuelt og en besked ad gangen. Her er nogle projekter:
Jeg kan kun finde ud af Python, og ved hvordan jeg sætter et job op på min webserver, der kan gøre Python-programmer regelmæssigt (det skulle jeg nemlig bruge til https://wallnot.dk).
Så – med kæmpe hjælp fra koden til Net-Eboks af Dmitry Karasik – har jeg skrevet et Python-program, der videresender nye beskeder i E-boks til min mail.
Programmet fungerer, men det er ikke gennemtestet, og tager ikke højde for fejl, fx. at brugeren indtaster forkerte oplysninger i programmet. Så det er nok en god idé også at logge ind på E-boks en gang imellem og lige se at alt bliver hentet og videresendt.
Du er velkommen til at bruge programmet, videreudvikle, og hvad du ellers kan finde på.
# -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com.
# Based on https://github.com/dk/Net-Eboks perl API for eboks.dk by Dmitry Karasik. Thanks!
""" This program logs on to e-boks.dk and takes new messages and sends them
to an e-mail. It requires mobile app login for e-boks (see http://www.e-boks.dk/help.aspx?pageid=db5a89a1-8530-418a-90e9-ff7f0713784a for
how to create). It also requires access to a secure (SSL) SMTP server and mail
account for sending e-mails. """
# Necessary modules
from datetime import datetime # Current date and time
import requests # Communicating with E-boks
import hashlib # Hash configuration for challenge/logon
import xml.etree.ElementTree as ET # Parse E-boks XML responses
import smtplib # Sending e-mails
from email.mime.multipart import MIMEMultipart # Creating multipart e-mails
from email.mime.text import MIMEText # Attaching text to e-mails
from email.mime.application import MIMEApplication # Attaching pdf to e-mails
from email.mime.image import MIMEImage # Attaching images to e-mails
from email.utils import formataddr # Used for correct encoding of senders with special characters in name (e.g. Københavns Kommune)
import chardet # Text message character set detection
import time # Pause between e-mails sent
# Configuration data
data = {
'emailserver': '', # Your mail server hostname: host.server.dk
'emailserverport': , # Mail server port, e.g. 465
'emailusername': '', # Sender mail account username
'emailpassword': '', # Sender mail account password
'emailfrom': '', # Sender e-mail, e.g. trump@usa.gov
'emailto': '', # Recipient e-mail, e.g. hillary@clinton.net
'cpr': '', # CPR number (no hyphens), e.g. 1234567890
'password': '', # E-boks mobile account password
'activation': '', # E-boks mobile account activation code
'numberofmessagesperfolder': '10', # Number of messages to request (10 is usually enough)
'unreadstatusvalue': "true", # Normally "true". If "false" also read messages are sent
'unreadmorethan': 0, # Normally 0, only unread messages are sent. If -1 all messages are sent
'sendemails': True, # If True, e-mails are sent, if False, they are not
'country': 'DK',
'type': 'P',
'deviceid': 'python-e-boks-000000000000',
'datetime': '',
'root': 'rest.e-boks.dk',
'nonce': '',
'sessionid': '',
'response': '3a1a51f235a8bd6bbc29b2caef986a1aeb77018d60ffdad9c5e31117e7b6ead3',
'uid': '',
'uname': '',
'challenge': ''
}
# Gets current date and time for E-boks challenge
now = datetime.now()
data['datetime'] = datetime.strftime(now, '%Y-%m-%d %H:%M:%SZ')
# Hashes parts of configuration data and sets challenge value to authenticate with E-boks
hashstring = data['activation']+":"+data['deviceid']+":"+data['type']+":"+data['cpr']+":"+data['country']+":"+data['password']+":"+data['datetime']
hashstringcoded = hashstring.encode('utf-8')
data['challenge'] = hashlib.sha256(hashstringcoded).hexdigest().encode('utf-8')
data['challenge'] = hashlib.sha256(data['challenge']).hexdigest()
# These functions are used to create sessionid, nonce and authstring values for communicating
# with E-boks throughout the program
def sessionid(authenticate):
sessionstart = authenticate.find('sessionid="')+len('sessionid="')
sessionend = authenticate.find('"', sessionstart)
data['sessionid'] = authenticate[sessionstart:sessionend]
def nonce(authenticate):
noncestart = authenticate.find('nonce="')+len('nonce="')
nonceend = authenticate.find('"', noncestart)
data['nonce'] = authenticate[noncestart:nonceend]
def createauthstring():
authstr = 'deviceid="' + data['deviceid'] + '",nonce="' + data['nonce'] + ',sessionid="' + data['sessionid'] + '",response="' + data['response'] + '"'
return authstr
# Logon to mail server
server = smtplib.SMTP_SSL(data['emailserver'], data['emailserverport'])
server.login(data['emailusername'], data['emailpassword'])
# First logon to e-boks
url = "https://" + data['root'] + "/mobile/1/xml.svc/en-gb/session"
content = '<Logon xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns="urn:eboks:mobile:1.0.0"><User identity="' + data['cpr'] + '" identityType="' + data['type'] + '" nationality="' + data['country'] + '" pincode="' + data['password'] + '"/></Logon>'
authstr = 'logon ' + 'deviceid="' + data['deviceid']+ '",' + 'datetime="' + data['datetime'] + '",' + 'challenge="' + data['challenge'] + '"'
headers = {
'Content-Type': 'application/xml',
'Content-Length': str(len(content)),
'X-EBOKS-AUTHENTICATE': authstr,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Accept-Encoding': 'gzip,deflate',
'Host': data['root'],
}
r = requests.put(url, headers=headers, data=content)
authenticate = r.headers['X-EBOKS-AUTHENTICATE']
nonce(authenticate)
sessionid(authenticate)
xml = ET.fromstring(r.text)
# Saves username and user id
data['uname'] = xml[0].attrib['name']
data['uid'] = xml[0].attrib['userId']
# Get folder data from e-boks
url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folders'
authstr = createauthstring()
headers = {
'X-EBOKS-AUTHENTICATE': authstr,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Host': data['root'],
}
r = requests.get(url, headers=headers)
authenticate = r.headers['X-EBOKS-AUTHENTICATE']
nonce(authenticate)
xml = ET.fromstring(r.text)
eboks_folders = xml
# Get folder id's and numbers of unread messages
for folder in eboks_folders:
folderid = folder.attrib['id']
unread = folder.attrib['unread']
# Get messages ONLY if any unread messages in folder
if int(unread) > data['unreadmorethan']: # Usually > 0. Can be changed to == 0 for debugging purposes
# Get list of messages
url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid
authstr = createauthstring()
headers = {
'X-EBOKS-AUTHENTICATE': authstr,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Host': data['root'],
}
params = {
'skip': '0',
'take': data['numberofmessagesperfolder']
}
r = requests.get(url, headers=headers, params=params)
authenticate = r.headers['X-EBOKS-AUTHENTICATE']
nonce(authenticate)
xml = ET.fromstring(r.text)
eboks_messages = xml
i = 0
max = int(params['take']) - 1
while i <= max:
for child in eboks_messages:
messageid = child[i].attrib['id']
subject = child[i].attrib['name']
sender = child[i][0].text
unreadstatus = child[i].attrib['unread']
attachmentcount = child[i].attrib['attachmentsCount']
format = child[i].attrib['format'].lower()
received = child[i].attrib['receivedDateTime']
i += 1
# Get only messages that are unread
if unreadstatus == data['unreadstatusvalue']: # Usually true. Can be changed to false for debugging purposes
# Start e-mail
msg = MIMEMultipart()
msg['From'] = formataddr((sender, data['emailfrom']))
msg['To'] = data['emailto']
msg['Subject'] = "E-boks: " + subject
body = ""
# Get message (marks it as read)
url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + messageid
authstr = createauthstring()
headers = {
'X-EBOKS-AUTHENTICATE': authstr,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Host': data['root'],
}
r = requests.get(url, headers=headers)
authenticate = r.headers['X-EBOKS-AUTHENTICATE']
nonce(authenticate)
# Get primary message content
url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + messageid + '/content'
authstr = createauthstring()
headers = {
'X-EBOKS-AUTHENTICATE': authstr,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Host': data['root'],
}
r = requests.get(url, headers=headers)
authenticate = r.headers['X-EBOKS-AUTHENTICATE']
nonce(authenticate)
# Attach primary message content to e-mail
if format in ("txt","text","plain"):
characterset = chardet.detect(r.content)
r.encoding = characterset['encoding']
body = r.text
msg.attach(MIMEText(body, 'plain'))
elif format in ("html","htm"):
characterset = chardet.detect(r.content)
r.encoding = characterset['encoding']
body = r.text
msg.attach(MIMEText(body, 'html'))
elif format == "pdf":
filename = "".join([c for c in subject if c.isalpha() or c.isdigit() or c==' ']).rstrip() + "." + format
part = MIMEApplication(r.content)
part.add_header('Content-Disposition', 'attachment', filename = filename)
msg.attach(part)
elif format in ("gif","jpg","jpeg","tiff","tif","webp"):
filename = "".join([c for c in subject if c.isalpha() or c.isdigit() or c==' ']).rstrip() + "." + format
part = MIMEImage(r.content)
part.add_header('Content-Disposition', 'attachment', filename = filename)
msg.attach(part)
# Get attachment data if message has attachments
if int(attachmentcount) > 0:
url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + messageid
authstr = createauthstring()
headers = {
'X-EBOKS-AUTHENTICATE': authstr,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Host': data['root'],
}
r = requests.get(url, headers=headers)
authenticate = r.headers['X-EBOKS-AUTHENTICATE']
nonce(authenticate)
xml = ET.fromstring(r.text)
eboks_attachment = xml
# Gets if, name and format of attachment
for child in eboks_attachment:
for subtree in child:
attachmentid = subtree.attrib['id']
attachmenttitle = subtree.attrib['name']
attachmentformat = subtree.attrib['format']
# Gets the actual attachment
url = 'https://' + data['root'] + '/mobile/1/xml.svc/en-gb/' + data['uid'] + '/0/mail/folder/' + folderid + '/message/' + attachmentid + '/content'
authstr = createauthstring()
headers = {
'X-EBOKS-AUTHENTICATE': authstr,
'Accept': '*/*',
'Accept-Language': 'en-US',
'Host': data['root'],
}
r = requests.get(url, headers=headers)
authenticate = r.headers['X-EBOKS-AUTHENTICATE']
nonce(authenticate)
# Attach attachment to e-mail
if attachmentformat in ("txt","text","html","htm","plain"):
filename = "".join([c for c in attachmenttitle if c.isalpha() or c.isdigit() or c==' ']).rstrip() + "." + attachmentformat
r.encoding = "utf-8"
part = MIMEText(r.text)
part.add_header('Content-Disposition', 'attachment', filename = filename)
msg.attach(part)
elif attachmentformat == "pdf":
filename = "".join([c for c in attachmenttitle if c.isalpha() or c.isdigit() or c==' ']).rstrip() + "." + attachmentformat
part = MIMEApplication(r.content)
part.add_header('Content-Disposition', 'attachment', filename = filename)
msg.attach(part)
elif attachmentformat in ("gif","jpg","jpeg","tiff","tif","webp"):
filename = "".join([c for c in attachmenttitle if c.isalpha() or c.isdigit() or c==' ']).rstrip() + "." + attachmentformat
part = MIMEImage(r.content)
part.add_header('Content-Disposition', 'attachment', filename = filename)
msg.attach(part)
# Send e-mail
if data['sendemails'] == True:
print("sending")
msg.attach(MIMEText(body, 'plain'))
server.sendmail(data['emailfrom'], data['emailto'], msg.as_string())
time.sleep(2)
Opdatering 18/02/2023: Nordnet ændrer tit på deres ting. På https://github.com/helmstedt/nordnet-utilities forsøger jeg at følge med, så hent gerne din kode der, hvis koden neden for ikke virker længere.
OPDATERING: Nordnet er ude i en ny version, som gør at man foreløbig er nødt til at ændre URLs i programmet til “classic.nordnet.dk” for at bruge den gamle version. På et tidspunkt virker det nok heller ikke længere. Jeg har opdateret koden neden for med den korrekte url.
Jeg kan godt lide at bruge Excel til at holde øje med min økonomi, så jeg har et ark med en pivottabel, som jeg bruger til at få overblik over min portefølje hos Nordnet. Nordnet har en funktion til at trække en CSV-fil med en oversigt over mine transaktioner, men det er lidt besværligt at skulle a) logge ind på Nordnet for derefter b) at gå ind på hver enkelt depot og c) trække en ny oversigt og copy/paste hver gang, der fx udbetales udbytte.
Her klikker du hos Nordnet for at generere en CSV-fil, du kan bruge til Excel
Derfor tænkte jeg: Kan jeg automatisere dette udtræk, sådan jeg altid har opdateret data i mit Excelark? Ja, det kan jeg. Med Python. Her fortæller jeg om hvordan og deler min kode.
Hvordan snakker man http med Nordnet?
Det første jeg gjorde, var at undersøge hvad der egentlig sker, når jeg beder Nordnet om en transaktionsfil. Det gør jeg i Chrome ved at trykke F12, vælge Network og undersøge hvad min browser sender af sted for at få en CSV-fil tilbage. Jeg kan se, at der ryger en cookie af sted og nogle parametre, der handler om bl.a. sortering og periode for de transaktioner, jeg vil have ud:
I Python bruger jeg modulet Requests til at snakke med Nordnet og forsøger at konstruere noget der ligner det, min browser smider af sted. Efter at have prøvet mig frem, finder jeg ud af, at det er den cookie, der hedder NOW, der er afgørende for, at modtage noget fra Nordnet. Jeg laver en cookie-ordbog, der foreløbig indeholder min NOW-værdi fra Chrome:
cookies = {'NOW': '63261fc324153bd1632006105c5b4444d97fc72a'}
Min forespørgsel, der giver mig data tilbage, ser sådan ud:
# LOGIN TO NORDNET #
# First part of cookie setting prior to login
url = 'https://www.nordnet.dk/mux/login/start.html?cmpi=start-loggain&state=signin'
r = requests.get(url)
cookies['LOL'] = r.cookies['LOL']
cookies['TUX-COOKIE'] = r.cookies['TUX-COOKIE']
# Second part of cookie setting prior to login
url = 'https://www.nordnet.dk/api/2/login/anonymous'
r = requests.post(url, cookies=cookies)
cookies['NOW'] = r.cookies['NOW']
# Actual login that gets us cookies required for primary account extraction
url = "https://www.nordnet.dk/api/2/authentication/basic/login"
r = requests.post(url,cookies=cookies, data = {'username': user, 'password': password})
cookies['NOW'] = r.cookies['NOW']
cookies['xsrf'] = r.cookies['xsrf']
Når denne procedure er gennemført, kan jeg med den genererede NOW-værdi trække transaktioner ud af mit primære depot (det jeg oprettede først, da jeg fik en Nordnet-konto).
For at trække transaktioner ud fra andre depoter, undersøger jeg hvad der sker, når jeg vælger et andet depot i Nordnet. Det sker ved at forespørge https://www.nordnet.dk/mux/ajax/session/bytdepa.html med mine gemte cookies og værdien fra den cookie, der hedder xrsf i headeren. Tilbage får jeg en ny NOW-værdi, som jeg kan bruge til at hente transaktioner på det andet depot, og en ny xrsf-værdi, som jeg kan bruge, hvis jeg har endnu flere depoter, jeg får brug for at skifte til:
# Switch to secondary account and set new cookies
url = 'https://www.nordnet.dk/mux/ajax/session/bytdepa.html'
headers = {'X-XSRF-TOKEN': cookies['xsrf']}
r = requests.post(url,cookies=cookies, headers=headers, data = {'portfolio': item['id']})
cookies['NOW'] = r.cookies['NOW']
cookies['xsrf'] = r.cookies['xsrf']
Til sidst skal jeg finde en fornuftig struktur for mit program, finde ud af hvordan jeg får lavet en god struktur i min CSV-fil (nogle gange returnerer Nordnet en CSV-fil med en kolonne for meget). Og så har jeg brug for en mulighed for at tilføje manuelle linjer til min CSV-fil (fordi jeg gerne vil have historisk data med fra et gammelt depot hos en anden bank).
Det færdige program i Python
Her er det færdige program. Du er velkommen til at bruge det, videreudvikle, osv.
<pre class="wp-block-syntaxhighlighter-code"><p># -*- coding: utf-8 -*-
# Author: Morten Helmstedt. E-mail: helmstedt@gmail.com
""" This program logs into a Nordnet account and extracts transactions as a csv file.
Handy for exporting to Excel with as few manual steps as possible """
import requests
from datetime import datetime
from datetime import date
import os
# USER ACCOUNT, PORTFOLIO AND PERIOD DATA. SHOULD BE EDITED FOR YOUR NEEDS #
# Nordnet user account credentials and name of primary portfolio (first one listed in Nordnet)
user = ''
password = ''
primaryportfolioname = "Frie midler"
# Names and portfolio ids for all any all secondary portfolios. The id is listed in
# Nordnet when selecting a portfolio. If no secondary portfolios the variable
# secondaryportfolioexists should be set to False.
secondaryportfolioexists = True
secondaryportfolios = [
{'name': 'Ratepension', 'id': ''},
]
# Start date (start of period for transactions) and date today used for extraction of transactions
startdate = '01.01.2013'
today = date.today()
enddate = datetime.strftime(today, '%d.%m.%Y')
# Manual date lines. These can be used if you have portfolios elsewhere that you would
# like to add manually to the data set. If no manual data the variable manualdataexists
# should be set to False
manualdataexists = True
manualdata = """
Id;Bogføringsdag;Handelsdag;Valørdag;Transaktionstype;Værdipapirer;Instrumenttyp;ISIN;Antal;Kurs;Rente;Afgifter;Beløb;Valuta;Indkøbsværdi;Resultat;Totalt antal;Saldo;Vekslingskurs;Transaktionstekst;Makuleringsdato;Verifikations-/Notanummer;Depot
;30-09-2013;30-09-2013;30-09-2013;KØBT;Obligationer 3,5%;Obligationer;;72000;;;;-69.891,54;DKK;;;;;;;;;Frie midler
"""
# CREATE OUTPUT FOLDER AND VARIABLES FOR LATER USE. #
# Checking that we have an output folder to save our csv file
if not os.path.exists("./output"):
os.makedirs("./output")
# Creates a dictionary to use with cookies
cookies = {}
# A variable to store transactions before saving to csv
transactions = ""
# Payload for transaction requests
payload = {
'year': 'all',
'month': 'all',
'trtyp': 'all',
'vp': 'all',
'curr': 'all',
'sorteringsordning': 'fallande',
'sortera': 'datum',
'startperiod': startdate,
'endperiod': enddate
}
# LOGIN TO NORDNET #
# First part of cookie setting prior to login
url = 'https://classic.nordnet.dk/mux/login/start.html?cmpi=start-loggain&state=signin'
r = requests.get(url)
cookies['LOL'] = r.cookies['LOL']
cookies['TUX-COOKIE'] = r.cookies['TUX-COOKIE']
# Second part of cookie setting prior to login
url = 'https://classic.nordnet.dk/api/2/login/anonymous'
r = requests.post(url, cookies=cookies)
cookies['NOW'] = r.cookies['NOW']
# Actual login that gets us cookies required for primary account extraction
url = "https://classic.nordnet.dk/api/2/authentication/basic/login"
r = requests.post(url,cookies=cookies, data = {'username': user, 'password': password})
cookies['NOW'] = r.cookies['NOW']
cookies['xsrf'] = r.cookies['xsrf']
# GET PRIMARY ACCOUNT TRANSACTION DATA #
# Get CSV for primary account
url = "https://classic.nordnet.dk/mux/laddaner/transaktionsfil.html"
data = requests.get(url, params=payload, cookies=cookies)
result = data.text
result = result.splitlines()
firstline = 0
for line in result:
if line and firstline == 0:
transactions += line + ';' + "Depot" + "\n"
firstline = 1
elif line:
# Sometimes Nordnet inserts one semicolon too many in the file. This removes the additional semicolon
if line.count(';') == 22:
position = line.rfind(';')
line = line [:position] + line[position+1:]
transactions += line + ';' + primaryportfolioname + "\n"
# GET TRANSACTION DATA FOR ALL/ANY SECONDARY ACCOUNTS #
if secondaryportfolioexists == True:
for item in secondaryportfolios:
# Switch to secondary account and set new cookies
url = 'https://classic.nordnet.dk/mux/ajax/session/bytdepa.html'
headers = {'X-XSRF-TOKEN': cookies['xsrf']}
r = requests.post(url,cookies=cookies, headers=headers, data = {'portfolio': item['id']})
cookies['NOW'] = r.cookies['NOW']
cookies['xsrf'] = r.cookies['xsrf']
# Get CSV for secondary account
url = "https://classic.nordnet.dk/mux/laddaner/transaktionsfil.html"
data = requests.get(url, params=payload, cookies=cookies)
result = data.text
result = result.split("\n",1)[1]
result = result.splitlines()
for line in result:
if line:
# Sometimes Nordnet inserts one semicolon too many in the file. This removes the additional semicolon
if line.count(';') == 22:
position = line.rfind(';')
line = line [:position] + line[position+1:]
transactions += line + ';' + item['name'] + "\n"
if manualdataexists == True:
manualdata = manualdata.split("\n",2)[2]
transactions += manualdata
# WRITE CSV OUTPUT TO FILE #
with open("./output/trans.csv", "w", encoding='utf8') as fout:
fout.write(transactions)</p></pre>
