{
"cells": [
{
"cell_type": "code",
"execution_count": 132,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"GECKO_TERMINAL = \"https://app.geckoterminal.com/api/p1/pools\"\n",
"\n",
"def page_count():\n",
" x = requests.get(GECKO_TERMINAL).json()\n",
" return x['links']['last']['meta']['series'][-1]\n",
"\n",
"\n",
"def page_result(page_number):\n",
" return requests.get(\n",
" GECKO_TERMINAL,\n",
" {\"include\": \"dex,dex.network,pool_metric,tokens\",\n",
" \"page\": page_number,\n",
" # \"include_network_metrics\": True\n",
" }\n",
" ).json()\n",
"\n",
"\n",
"r = []\n",
"# for page_number in range(1, page_count()):\n",
"# try:\n",
"# r.append(page_result(page_number))\n",
"# except Exception as e:\n",
"# print(e)\n",
"\n",
"for page_number in range(1, 2):\n",
" r.append(page_result(page_number))\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import json \n",
"with open(\"gecko_terminal_pools.json\") as f: \n",
" r = json.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"80"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(r)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"({'dex', 'network', 'pool', 'pool_metric', 'token'}, 31215)"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dexs = {}\n",
"tokens = {}\n",
"network = {}\n",
"pool_metric = {}\n",
"pools = {}\n",
"dex_metrics ={}\n",
"rel_types = [\"dex\", \"token\",\"network\",\"pool_metric\", \"pool\"]\n",
"\n",
"results = {\n",
" \"pool\":pools,\n",
" \"dex\": dexs,\n",
" \"token\": tokens,\n",
" \"network\": network,\n",
" \"pool_metric\": pool_metric,\n",
" \"dex_metric\":dex_metrics,\n",
"}\n",
"count = 0\n",
"_types = set()\n",
"for result in r:\n",
" for element in [*result['data'], *result['included']]:\n",
" count +=1\n",
" _types.add(element['type'])\n",
" match element['type']:\n",
" case \"dex\":\n",
" dexs[element['id']] = element\n",
" case \"dex_metric\":\n",
" dex_metrics[element['id']] = element\n",
" case \"pool\":\n",
" pools[element['id']] = element\n",
" case \"token\":\n",
" tokens[element['id']] = element\n",
" case \"network\":\n",
" network[element['id']] = element\n",
" case \"pool_metric\":\n",
" pool_metric[element['id']] = element\n",
"_types , count"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'pool': 7078,\n",
" 'dex': 270,\n",
" 'token': 4685,\n",
" 'network': 59,\n",
" 'pool_metric': 7078,\n",
" 'dex_metric': 0}"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"for _type in _types:\n",
" type_result=results[_type]\n",
" for type_id , type in type_result.items():\n",
" if 'relationships' not in type:\n",
" continue\n",
" for rel_type, rel in type['relationships'].items():\n",
" if rel_type in ['network_metric','dex_metric']:\n",
" continue\n",
" rel_data = rel['data']\n",
" if isinstance(rel_data, list):\n",
" for i, _elm in enumerate(rel_data):\n",
" _elem_id = _elm['id']\n",
" _elem_type = _elm['type']\n",
" if rel_type not in type_result[type_id] or isinstance(type_result[type_id].get(rel_type), list):\n",
" type_result[type_id][rel_type] = {}\n",
" \n",
" type_result[type_id][rel_type][_elem_id] = results[_elem_type][_elem_id]\n",
" else: \n",
" if rel_data['id'] not in results[rel_type]:\n",
" print(rel_data['id'], rel_type)\n",
" else:\n",
" type_result[type_id][rel_type] = results[rel_type][rel_data['id']]\n",
" for rel_type in rel_types:\n",
" if rel_type not in type_result[type_id]:\n",
" type_result[type_id][rel_type] = None\n",
"{k:len(v) for k,v in results.items()}\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'pool': 7078,\n",
" 'dex': 270,\n",
" 'token': 4685,\n",
" 'network': 59,\n",
" 'pool_metric': 7078,\n",
" 'dex_metric': 0}"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": []
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"import json \n",
"with open(\"tmp.json\",\"w+\") as f: json.dump(list(pools.values())[0],f)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"from pydantic import BaseModel\n",
"from typing import Optional, List, Set\n",
"import logging\n",
"\n",
"logger = logging.getLogger(__name__)\n",
"\n",
"\n",
"class Token(BaseModel):\n",
" chainId: int\n",
" address: str\n",
" name: str\n",
" symbol: str\n",
" decimals: int\n",
" priceUSD: Optional[str] # Later convert to float\n",
" tags: Optional[List[str]]\n",
" coingeckoId: Optional[str]\n",
" lifiId: Optional[str] # ! In lifi list is coinkey\n",
" listedIn: Optional[List[str]]\n",
" logoURI: Optional[str]\n",
" verify: bool = False\n",
"\n",
"class Pair(Token):\n",
" dex: str\n",
" tokens: List[Token]\n",
" totalSupply: Optional[int]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from web3 import Web3\n",
"valid_tokens = {}\n",
"\n",
"for i, (token_id,token) in enumerate(tokens.items()):\n",
" try:\n",
" valid_tokens[token['id']] = Token(\n",
" chainId=token['network']['attributes']['chain_id'],\n",
" address=Web3.toChecksumAddress(token['attributes']['address']),\n",
" symbol=token['attributes']['symbol'],\n",
" name=token['attributes']['name'],\n",
" decimals=-1,\n",
" logoURI=token['attributes']['image_url'],\n",
" )\n",
" except ValueError as e:\n",
" # print(token_id, token['network']['attributes']['chain_id'], e)\n",
" pass\n",
" except Exception as e:\n",
" print(i)\n",
" raise e\n",
"\n",
"print(f\"\\n\\n{len(valid_tokens)=} errors {len(tokens)- len(valid_tokens)=}\")"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"148390732 None '2984234'\n",
"148390726 None '2984234'\n",
"148914778 None '4045901'\n",
"148914793 None '4046257'\n",
"155124253 None '4053460'\n",
"148914862 None '4046026'\n",
"148914935 None '4047758'\n",
"148914884 None '4046258'\n",
"148914840 None '4046366'\n",
"148390826 None '2984234'\n",
"148914915 None '4046026'\n",
"148916424 None '4048260'\n",
"148915027 None '4052264'\n",
"148914906 None '4046257'\n",
"148917519 None '4047549'\n",
"148914861 None '4046924'\n",
"148915026 None '4047769'\n",
"148914920 None '4046258'\n",
"148914780 None '4045901'\n",
"148914779 None '4045999'\n",
"148595407 None '2989037'\n",
"148915235 None '4052264'\n",
"148390764 None '2984309'\n",
"148390788 None '2984338'\n",
"148915020 None '4055330'\n",
"148390803 None '2984369'\n",
"148390772 None '2984322'\n",
"149411800 None '5050866'\n",
"148595389 None '2989037'\n",
"148914881 None '4046661'\n",
"148914875 None '4047293'\n",
"148915054 None '4045828'\n",
"148914926 None '4046026'\n",
"148915199 None '4055934'\n",
"148391207 None '2984234'\n",
"148915408 None '4055396'\n",
"148914958 None '4050047'\n",
"148915208 None '4047912'\n",
"148922537 None '4046020'\n",
"148395286 None '2984322'\n",
"148914879 None '4046414'\n",
"148914961 None '4054927'\n",
"148915198 None '4046525'\n",
"148914827 None '4045783'\n",
"148915028 None '4047730'\n",
"148915272 None '4047728'\n",
"148917753 None '4047993'\n",
"148390768 None '2984309'\n",
"148914816 None '4046295'\n",
"148390741 None '2984258'\n",
"148597973 None '2989037'\n",
"148914866 None '4046836'\n",
"149863064 None '4056314'\n",
"148559174 None '2984258'\n",
"149823318 None '4051300'\n",
"148915958 None '4046683'\n",
"149012265 None '4053055'\n",
"148915096 None '4051243'\n",
"148914912 None '4046020'\n",
"154018958 None '4047047'\n",
"148537338 None '2984256'\n",
"148390758 None '2984298'\n",
"148393154 None '2989037'\n",
"148915500 None '4047735'\n",
"148390761 None '2984306'\n",
"148391097 None '2984234'\n",
"148915367 None '4053222'\n",
"153616413 None '4054223'\n",
"148537141 None '2984331'\n",
"149149379 None '4456561'\n",
"148914860 None '4046634'\n",
"149559153 None '4793199'\n",
"148390841 None '2984440'\n",
"148391155 None '2984369'\n",
"148666828 None '2989037'\n",
"148390797 None '2984338'\n",
"148914909 None '4046530'\n",
"148537264 None '2984309'\n",
"148390754 None '2984292'\n",
"148914852 None '4046296'\n",
"148914886 None '4047293'\n",
"148914824 None '4046284'\n",
"148390822 None '2984306'\n",
"154857578 None '4053553'\n",
"148537256 None '2984783'\n",
"148537262 None '2984783'\n",
"148391526 None '2985783'\n",
"148914910 None '4046369'\n",
"148391042 None '2984839'\n",
"148537443 None '2984326'\n",
"148393270 None '2984342'\n",
"148914913 None '4046026'\n",
"148390773 None '2984235'\n",
"148665193 None '2984256'\n",
"148917526 None '4048590'\n",
"155399691 None '13732776'\n",
"148391147 None '2985036'\n",
"148391235 None '2984234'\n",
"148538164 None '2984258'\n",
"149894277 None '5773517'\n",
"148916722 None '4046560'\n",
"148915721 None '4048720'\n",
"149322369 None '4597863'\n",
"148538044 None '2984256'\n",
"148919324 None '4047951'\n",
"148390737 None '2984235'\n",
"148390743 None '2984264'\n",
"148390776 None '2984326'\n",
"148391016 None '2984783'\n",
"148595647 None '2989037'\n",
"148537317 None '2984331'\n",
"148915652 None '4055613'\n",
"148390763 None '2984308'\n",
"148390790 None '2984342'\n",
"148917074 None '4048603'\n",
"148537783 None '2984331'\n",
"148433277 None '2984235'\n",
"153956951 None '4055954'\n",
"149347662 None '2984783'\n"
]
}
],
"source": [
"valid_pools = {}\n",
"\n",
"for i, (pool_id,pool) in enumerate(pools.items()):\n",
" try:\n",
" pool_tokens = [\n",
" valid_tokens[_] for _ in pool['tokens']\n",
" ]\n",
" \n",
" valid_pools[pool['id']] = Pair(\n",
" chainId=pool['dex']['network']['attributes']['chain_id'],\n",
" address=Web3.toChecksumAddress(pool['attributes']['address']),\n",
" symbol=pool['dex']['attributes']['identifier'] + \"LP (\" + pool['attributes']['name'] + \")\",\n",
" name=pool['attributes']['name'],\n",
" decimals=-1,\n",
" logoURI=pool['dex']['attributes']['image_url'],\n",
" dex=pool['dex']['attributes']['identifier'],\n",
" tokens=pool_tokens\n",
" )\n",
" except KeyError as e:\n",
" print(pool_id, pool['dex']['network']['attributes']['chain_id'], e)\n",
" pass\n",
" except ValueError as e:\n",
" pass\n",
" except Exception as e:\n",
" print(i)\n",
" raise e"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"6959"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(valid_pools)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"import functools\n",
"class SetEncoder(json.JSONEncoder):\n",
" def default(self, obj):\n",
" if isinstance(obj, BaseModel):\n",
" return obj.dict()\n",
" if isinstance(obj, set):\n",
" return list(obj)\n",
" return json.JSONEncoder.default(self, obj)\n",
"\n",
"\n",
"dump = functools.partial(json.dump, cls=SetEncoder)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"with open(\"raw.json\",\"w+\") as f:\n",
" dump(valid_pools,f)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"6959"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"all_pairs = [_ for _ in valid_pools.values()]\n",
"with open(\"all_pairs.json\",\"w+\") as f:\n",
" dump(all_pairs,f)\n",
"len(all_pairs)"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"6959"
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"chain_seperated ={}\n",
"count = 0\n",
"for pair in valid_pools.values():\n",
" pair: Pair\n",
" if pair.chainId not in chain_seperated:\n",
" chain_seperated[pair.chainId] = []\n",
" chain_seperated[pair.chainId] = pair\n",
" count += 1\n",
"\n",
"with open(\"pairs_chain_seperated.json\",\"w+\") as f:\n",
" dump(chain_seperated, f)\n",
"count "
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.10.8 ('venv': venv)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.8"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "6244d4230505c7e018fb33d93bb63bf28875b315caf110afd51c11ee9029a40e"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}