Verified Commit 508733d2 authored by Eliot Berriot's avatar Eliot Berriot 💬

Initial commit

parents
Pipeline #877 passed with stage
in 16 seconds
build_api:
# Simply publish the json file as an artifact
stage: build
image: python:3
artifacts:
name: "instances"
paths:
- data.json
script: python collect.py instances.txt data.json
only:
- master@funkwhale/instances
Funkwhale instances
===================
This repository aims to keep a list of funkwhale instances who opt-in
for statistic collections. Those instances are queried on a daily basis
to monitor the state of the network and to provide structured data
about them to third-party applications in form of a JSON file.
import argparse
import datetime
import json
import urllib.error
import urllib.parse
import urllib.request
class InvalidPayload(ValueError):
pass
class UnreachableInstance(Exception):
pass
def main():
parser = argparse.ArgumentParser()
parser.add_argument('list_file')
parser.add_argument('data_file')
args = parser.parse_args()
instances = get_instances(args.list_file)
print('Loaded {} instances from {}'.format(len(instances), args.list_file))
data = get_data(instances)
with open(args.data_file, 'w') as f:
f.write(json.dumps(data, sort_keys=True, indent=2))
def get_instances(list_file):
with open(list_file) as f:
return [
line.strip()
for line in f.readlines()
if line.strip() and not line.startswith('#')
]
def get_data(instances):
data = {
'instances': []
}
for i, url in enumerate(instances):
print('[{}/{}] fetching {}'.format(i+1, len(instances), url))
data['instances'].append(get_instance_data(url))
data['count'] = len(data['instances'])
return data
def get_instance_data(url):
stats_url = '{}/api/v1/instance/stats/'.format(url)
settings_url = '{}/api/v1/instance/settings/'.format(url)
parsed = urllib.parse.urlparse(url)
data = {
'name': parsed.netloc,
'fetchDate': datetime.datetime.now().isoformat(),
'info': {}
}
try:
settings = get_settings_data(settings_url)
stats = get_stats_data(stats_url)
except (InvalidPayload, UnreachableInstance) as e:
print('Error while fetching {}: {}'.format(url, e))
data['fetchSuccess'] = False
data['up'] = False
else:
data['fetchSuccess'] = True
data['up'] = True
data.update(stats)
data['openRegistrations'] = settings['openRegistrations']
data['info']['shortDescription'] = settings['shortDescription']
data['info']['fullDescription'] = settings['fullDescription']
return data
def get_settings_data(url):
data = request(url)
by_key = {s['identifier']: s for s in data}
try:
return {
'openRegistrations': by_key['users__registration_enabled']['value'],
'shortDescription': by_key['instance__short_description']['value'],
'fullDescription': by_key['instance__long_description']['value'],
}
except KeyError as e:
raise InvalidPayload(str(e))
def get_stats_data(url):
data = request(url)
try:
return {
'users': data['users'],
'tracks': data['tracks'],
'albums': data['albums'],
'artists': data['artists'],
'trackFavorites': data['track_favorites'],
'listenings': data['listenings'],
'musicDuration': data['music_duration'],
}
except KeyError as e:
raise InvalidPayload(str())
def request(url):
try:
req = urllib.request.Request(
url,
headers={
'User-Agent': 'funkwhale/instances bot <https://code.eliotberriot.com/funkwhale/instances>'})
with urllib.request.urlopen(req) as response:
assert response.status == 200, 'Wrong backend response ({})'.format(reponse.status)
return json.loads(response.read().decode('utf-8'))
except (AssertionError, ValueError, urllib.error.HTTPError, urllib.error.URLError) as e:
raise UnreachableInstance(str(e))
if __name__ == '__main__':
main()
{
"count": 3,
"instances": [
{
"albums": 1034,
"artists": 359,
"fetchDate": "2018-05-06T18:34:46.144042",
"fetchSuccess": true,
"info": {
"fullDescription": "Hi!\r\n\r\nI'm Eliot Berriot, the owner of this funkwhale instance. Funkwhale started as a personal project, in 2015, and I've been using it to stream my music eversince.\r\n\r\nI think it's time for me to open the project more, as well as this instance, so new users can try it, tell me what's working and what's not.\r\n\r\nIf you're interested in music or free and open-source software maybe you'll want to join us :)\r\n\r\n## Become a beta-tester\r\n\r\nYou don't need to be tech-savyy for helping, and it's really simple to get involved:\r\n\r\n1. [Create an account](https://funkwhale.eliotberriot.com/signup)\r\n2. Use the service\r\n3. Report any bug you encounterep as well as your imporvements ideas, if you have any\r\n\r\n## Getting in touch\r\n\r\nReporting a bug or suggesting a new feature is as simple as creating an issue on the [issue tracker](https://code.eliotberriot.com/funkwhale/funkwhale/issues).\r\n\r\nYou can also join the community on Matrix, where people will be able to answer your questions and welcome you. Simply join [#funkwhale:matrix.org](https://riot.im/app/#/room/#funkwhale:matrix.org).\r\n\r\nFinally, I'm personnaly reachable on Mastodon, at [@eliotberriot@mastodon.eliotberriot.com](https://mastodon.eliotberriot.com/@eliotberriot), as well as by email, at contact@eliotberriot.com. Feel free to say hi!\r\n\r\nThat's it, I hope you enjoy this project as well as I do :)",
"shortDescription": ""
},
"listenings": 24322,
"musicDuration": 654.9941666666666,
"name": "funkwhale.eliotberriot.com",
"openRegistrations": false,
"trackFavorites": 1000,
"tracks": 9452,
"up": true,
"users": 62
},
{
"albums": 7,
"artists": 7,
"fetchDate": "2018-05-06T18:34:46.501681",
"fetchSuccess": true,
"info": {
"fullDescription": "",
"shortDescription": ""
},
"listenings": 0,
"musicDuration": 0,
"name": "demo.funkwhale.audio",
"openRegistrations": false,
"trackFavorites": 0,
"tracks": 58,
"up": true,
"users": 1
},
{
"fetchDate": "2018-05-06T18:34:46.801183",
"fetchSuccess": false,
"info": {},
"name": "test.yolo.comf",
"up": false
}
]
}
\ No newline at end of file
https://funkwhale.eliotberriot.com
https://demo.funkwhale.audio
https://test.yolo.comf
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment