Internet on magnets 4 - Divide the magnet into pieces

In my articles on the Internet on Magnets, I suggested publishing universal magnets with which you can download a file from any p2p network ( Gnutella , Gnutella2 , Edonkey2000 , DirectConnect , BitTorrent ). They can be obtained either by using the rhash program, or by mixing different magnets and links to a file on the page of the Magnet Converter service. But there is a problem that not many clients for p2p networks are loyal to an arbitrary order of parameters in a magnet. I decided to write a script that corrects this situation on all pages of the Internet.

Under the cat you will find JavaScript code with comments and a test link.

So. Create a folder “magnet-converter” in which the files will be:

File " manifest.json ":

So that the script could be connected as a Chrome extension.
')

{ "manifest_version": 2, "name": "Magnet Splitter", "description": "Splits universal magnet links to suitable for different p2p clients", "version": "1.1", "content_scripts": [ { "matches": [ "http://*/*", "https://*/*" ], "js": [ "insert.js" ], "run_at": "document_end" } ], "web_accessible_resources": [ "magnet-converter.user.js" ] } 

File " insert.js ":

Insert the script on the page.

 var script = document.createElement('script'); script.type = 'text/javascript'; script.src = chrome.extension.getURL("magnet-converter.user.js"); script.async = 1; document.head.appendChild(script); 

File " magnet-converter.user.js "

The script itself. It can be connected as a custom script Greasemonkey and Tampermonkey. Also, the owner of the site that publishes universal magnets can connect it to their pages.

 // ==UserScript== // @name Magnet Splitter // @description Splits universal magnet links to suitable for different p2p clients. // @author ivan386 // @license MIT // @version 1.1 // @run-at document-end // @include http://*/* // @include https://*/* // ==/UserScript== javascript: 

/*

Bookmarklet

This script can be copied and saved to bookmarks (from “javascript:” to “void ();” inclusive).

For the script to work correctly in the bookmarklet:
1. use only inline comments
2. end instructions with a semicolon
3. put a space after else at the end of the line
*/

 (function() { 'use strict'; function try_decode(data){ try{ data = decodeURIComponent(data); }catch(e){console.error(e)} return data; } 

/*
We disassemble the magnet into pieces
*/

 function detect_ipfs_hash(url, file) { url.replace(/\/ipfs\/([123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz]+)/,function(all, hash){ if (!file.hash) file.hash = {}; file.hash.ipfs = hash; }); return; } function parse_urn(urn_name, urn_data, file) { if (!file.hash) file.hash = {}; switch (urn_name){ case "urn:sha1:": /* sha1   (Base32) */ file.hash.sha1 = urn_data; break; case "urn:ed2k:": /* ed2k   (Hex) */ case "urn:ed2khash:": file.hash.ed2k = urn_data; break; case "urn:aich:": /* Advanced Intelligent Corruption */ file.hash.aich = urn_data; break; case "urn:btih:": /* BitTorrent Info Hash (Hex, Base32) */ if (urn_data.length < 40) file.hash.btih = base32_to_hex(urn_data); else file.hash.btih = urn_data; break; case "urn:ipfs:": /* InterPlanetary File System  */ file.hash.ipfs = urn_data; break; case "urn:tree:tiger:": /* Tiger Tree Hash (TTH)  (Base32) */ file.hash.tree_tiger = urn_data; break; case "urn:bitprint:": /* [SHA1].[TTH] (Base32) */ var sha1_tth = urn_data.split("."); if (sha1_tth && sha1_tth.length == 2){ file.hash.sha1 = sha1_tth[0]; file.hash.tree_tiger = sha1_tth[1]; file.hash.bitprint = urn_data; }; break; default: /*     */ if (!file.urns) file.urns = []; file.urns.push(urn_name+urn_data); } } function parse_magnet(params, file){ if (!file) file = {}; params.replace(/([a-z0-9\.]+)=((([a-z0-9\.]+:)*)([^&]+))&?/gmi, function(all, name, data, urn, _, urn_data){ data = try_decode(data); switch (name){ case "dn": /* (Display Name) —  . */ file.name = data; break; case "xl": /* (eXact Length) —    . */ file.size = data; break; case "br": /*  -       . */ file.bitrate = data; break; case "tr": /* (TRacker) —    BitTorrent . */ if (!file.trackers) file.trackers = []; file.trackers.push(data); break; case "mt": file.collection = file.collection || []; file.collection.push(data); break; case "x.do": /*     . */ if (!file.description_url) file.description_url = []; file.description_url.push(data); break; case "fl": /*    . */ if (!file.torrent_file_url) file.torrent_file_url = []; file.torrent_file_url.push(data); break; case "as": /* (Acceptable Source) — -    . */ case "ws": /* Web Seed -        . */ if (!file.url) file.url = []; file.url.push(data); detect_ipfs_hash(data, file); break; case "xs": /* (eXact Source) —      P2P . */ if (!file.xurl) file.xurl = []; file.xurl.push(data); detect_ipfs_hash(data, file); break; case "xt": /* URN,   */ parse_urn(urn, urn_data, file); break; case "x.ed2k.p": (file.hash_list = file.hash_list || {}).ed2k = data.split(":"); break; } }); return file; } 

/*

Here the magnet is assembled for the torrent.

Torrent client needs first of all bittorrent info hash (btih) .
It follows first and is required for this type of magnet.

Magnet example:
magnet:?xt=urn:btih:16253d9beb0df49fe30bacc62ea10ba63939f0f8&dn=ruwiki-20141114-pages-meta-current.xml.bz2&tr=http%3A%2F%2Furl2torrent.net%3A6970%2Fannounce&ws=http%3A%2F%2Fdumps.wikimedia.org%2Fruwiki%2F20141114%2Fruwiki-20141114-pages-meta-current.xml.bz2&fl=http://torcache.net/torrent/16253D9BEB0DF49FE30BACC62EA10BA63939F0F8.torrent

*/

  function torrent_magnet(file){ var magnet = ["magnet:?"]; if (file && file.hash && file.hash.btih) { magnet.push("xt=urn:btih:"); magnet.push(encodeURIComponent(file.hash.btih)); }else return undefined; if (file.name) { magnet.push("&dn="); magnet.push(encodeURIComponent(file.name)); } if (file.trackers) { for (var i=0; i < file.trackers.length; i++){ magnet.push("&tr="); magnet.push(encodeURIComponent(file.trackers[i])); } } if (file.url){ for (var i=0; i < file.url.length; i++){ magnet.push("&ws="); magnet.push(encodeURIComponent(file.url[i])); } } if (file.torrent_file_url){ for (var i=0; i < file.torrent_file_url.length; i++){ magnet.push("&fl="); magnet.push(file.torrent_file_url[i]); } } return magnet.join(""); } 

/*

This feature collects a magnet for DirectConnect (DC ++) clients.

For them, tree tiger hash (TTH) is required. Some clients understand urn: bitprint which also contains TTH but urn: tree: tiger: must understand everything.
If there are no other parameters, DC ++ will open a TTH file search window.

Magnet example:
magnet:?xt=urn:tree:tiger:JNOANHGPELY63I2OMSPQ3J73AS2P4AWB5MTBJCQ&xl=2981763794&dn=ruwiki-20141114-pages-meta-current.xml.bz2&xs=http://cache.freebase.be/u24iyrp3wtlry5kjh7sacblft2lam62u&x.do=https%3A%2F%2Fdumps.wikimedia.org%2Fruwiki%2F20141114%2F&xs=dchub://allavtovo.ru

*/

  function dc_magnet(file){ var magnet = ["magnet:?"]; if (file && file.hash && file.hash.tree_tiger) { magnet.push("xt=urn:tree:tiger:"); magnet.push(file.hash.tree_tiger); }else return; if (file.size) { magnet.push("&xl="); magnet.push(encodeURIComponent(file.size)); } if (file.name) { magnet.push("&dn="); magnet.push(encodeURIComponent(file.name)); } if (file.xurl){ for (var i=0; i < file.xurl.length; i++){ magnet.push("&xs="); magnet.push(file.xurl[i]); } } if (file.description_url){ for (var i=0; i < file.description_url.length; i++){ magnet.push("&x.do="); magnet.push(encodeURIComponent(file.description_url[i])); } } return magnet.join(""); } 

/*

Here is going to ed2k link.

At the moment, the magnets from Edonkey2000 customers understood only aMule . Therefore, we are collecting a standard link for this network ed2k.

Link example:
ed2k://|file|ruwiki-20141114-pages-meta-current.xml.bz2|2981763794|0218392e98873112284de6913efee0df|s=http%3A%2F%2Fdumps.wikimedia.org%2Fruwiki%2F20141114%2Fruwiki-20141114-pages-meta-current.xml.bz2|/

*/

  function ed2k_link(file){ var link = ["ed2k://|file"]; if (file && file.hash && file.name && file.size && file.hash.ed2k) { link.push(encodeURIComponent(file.name)); link.push(file.size); link.push(file.hash.ed2k); if (file.hash.aich) link.push("h=" + file.hash.aich); if (file.hash_list && file.hash_list.ed2k) link.push("p=" + file.hash_list.ed2k.join(":")); if (file.url) for (var i=0; i < file.url.length; i++) link.push("s=" + encodeURIComponent(file.url[i])); if (file.collection) for (var i=0; i < file.collection.length; i++) link.push("f=" + encodeURIComponent(file.collection[i])); if (file.xurl){ var sources = []; file.xurl.map( function (xurl){ var matches = (/ed2kftp\:\/\/([^\/]+)\/[a-z0-9]+\/[0-9]+\//gmi).exec(xurl); if (matches) sources.push(matches[1]); } ); if ( sources.length > 0 ) link.push("/|sources," + sources.join(",")); } link.push("/"); return link.join("|"); } } 

/*

Full magnet

This magnet can be used by any p2p client if it contains the hashes it needs. But only Shareaza is coping with this task.

Magnet example:
magnet:?xt=urn:ed2k:0218392e98873112284de6913efee0df&xl=2981763794&dn=ruwiki-20141114-pages-meta-current.xml.bz2&xt=urn:bitprint:U24IYRP3WTLRY5KJH7SACBLFT2LAM62U.JNOANHGPELY63I2OMSPQ3J73AS2P4AWB5MTBJCQ&xt=urn:btih:16253d9beb0df49fe30bacc62ea10ba63939f0f8&tr=http%3A%2F%2Furl2torrent.net%3A6970%2Fannounce&ws=http%3A%2F%2Fdumps.wikimedia.org%2Fruwiki%2F20141114%2Fruwiki-20141114-pages-meta-current.xml.bz2&x.do=https%3A%2F%2Fdumps.wikimedia.org%2Fruwiki%2F20141114%2F&fl=http%3A%2F%2Ftorcache.net%2Ftorrent%2F16253D9BEB0DF49FE30BACC62EA10BA63939F0F8.torrent&xs=http://cache.freebase.be/u24iyrp3wtlry5kjh7sacblft2lam62u&xs=dchub://allavtovo.ru

*/

  function full_magnet(file, as){ var magnet = ["magnet:?"]; var amp = false; if (!file) return; if (file.hash){ if (file.hash.btih){ magnet.push("xt=urn:btih:"); magnet.push(file.hash.btih); amp = true; } if (file.hash.sha1 && file.hash.tree_tiger){ if (amp) magnet.push("&"); else amp = true; magnet.push("xt=urn:bitprint:"); magnet.push(file.hash.sha1); magnet.push("."); magnet.push(file.hash.tree_tiger); }else if(file.hash.sha1){ if (amp) magnet.push("&"); else amp = true; magnet.push("xt=urn:sha1:"); magnet.push(file.hash.sha1); }else if(file.hash.tree_tiger){ if (amp) magnet.push("&"); else amp = true; magnet.push("xt=urn:tree:tiger:"); magnet.push(file.hash.tree_tiger); } if (file.hash.ed2k){ if (amp) magnet.push("&"); else amp = true; magnet.push("xt=urn:ed2k:"); magnet.push(file.hash.ed2k); } } if (file.size) { if (amp) magnet.push("&"); else amp = true; magnet.push("xl="); magnet.push(encodeURIComponent(file.size)); } if (file.name) { if (amp) magnet.push("&"); else amp = true; magnet.push("dn="); magnet.push(encodeURIComponent(file.name)); } if (file.hash){ if (file.hash.aich){ if (amp) magnet.push("&"); else amp = true; magnet.push("xt=urn:aich:"); magnet.push(file.hash.aich); } if (file.hash.ipfs){ if (amp) magnet.push("&"); else amp = true; magnet.push("xt=urn:ipfs:"); magnet.push(file.hash.ipfs); } } if (file.urns){ file.urns.map((urn)=>{ if (amp) magnet.push("&"); else amp = true; magnet.push("xt="); magnet.push(urn); }) } if (file.trackers){ for (var i=0; i < file.trackers.length; i++){ if (amp) magnet.push("&"); else amp = true; magnet.push("tr="); magnet.push(encodeURIComponent(file.trackers[i])); } } if (file.url){ for (var i=0; i < file.url.length; i++){ if (amp) magnet.push("&"); else amp = true; if (as || !(file.hash && file.hash.btih)) magnet.push("as="); else magnet.push("ws="); magnet.push(encodeURIComponent(file.url[i])); } } if (file.bitrate){ if (amp) magnet.push("&"); else amp = true; magnet.push("br="); magnet.push(encodeURIComponent(file.bitrate)); } if (file.description_url){ for (var i=0; i < file.description_url.length; i++){ if (amp) magnet.push("&"); else amp = true; magnet.push("x.do="); magnet.push(encodeURIComponent(file.description_url[i])); } } if (file.torrent_file_url){ for (var i=0; i < file.torrent_file_url.length; i++){ if (amp) magnet.push("&"); else amp = true; magnet.push("fl="); magnet.push(encodeURIComponent(file.torrent_file_url[i])); } } if (file.collection){ for (var i=0; i < file.collection.length; i++){ if (amp) magnet.push("&"); else amp = true; magnet.push("mt="); magnet.push(encodeURIComponent(file.collection[i])); } } if (file.hash_list && file.hash_list.ed2k){ if (amp) magnet.push("&"); else amp = true; magnet.push("x.ed2k.p="); magnet.push(file.hash_list.ed2k.join(":")); } if (file.xurl){ for (var i=0; i < file.xurl.length; i++){ if (amp) magnet.push("&"); else amp = true; magnet.push("xs="); magnet.push(file.xurl[i]); } } if ( magnet.length > 1 ) return magnet.join(""); } 

/*
In the past, Base32 btih encoding was used in torrent magnets. Now used HEX . Shareaza uses Base32 in its old-fashioned way and this function converts btih back to HEX.
*/

  function base32_to_hex(base32){ /* http://tools.ietf.org/html/rfc4648 */ if (!base32) return ""; var number = 0; var bit_pos = 0; var detector = /[2-7a-z]/gmi; var hex_str = []; base32.replace(detector, function(letter){ var val = parseInt(letter, 36); if (val <= 7) val += 24; else val -= 10; number = ((number << 5) | val) & 255; bit_pos += 5; for (; bit_pos >= 4;){ bit_pos -= 4; var hex_num = (number >> bit_pos) & 15; hex_str.push(hex_num.toString(16)); } }); return hex_str.join(""); } 

/*

Bonus: Micro Torrent

If the universal magnet has a sha1 hash using it we can generate a micro torrent file. This is a torrent in which only one hour which includes the entire file. The sha1 hash of this part is identical in sha1 to the hash of the file itself. This torrent can be obtained from a magnet that allows you to fumble small files on the BitTorrent network without the need to create a full torrent file. Also, a micro torrent can be transferred via the BitTorrent network to a large torrent file completely and not just the info part.
*/

 /*     */ function raw_length(encodedURI){ var count = 0; var index = 0; for (index = encodedURI.indexOf("%"); index>=0 && (++count) ; index = encodedURI.indexOf("%", index+1)); return encodedURI.length-count*2; } /*  bencode   */ function push_string(string, to_list){ to_list.push(raw_length(string)); to_list.push(":"); to_list.push(string); } 

/*

Function build micro torrent file.

In order to collect it we need: sha1 file hash, file size, file name. Parameters in the info part of the micro torrent are written alphabetically by parameter name. This reduces the variability of the assembly torrent file and increases the likelihood of obtaining the same btih from the same magnet. UTorrent has a limit on the minimum size of the part, so if the file is less than 16384 bytes, the size of the part is set equal to this number.

Example:
data:application/x-bittorrent;,d4:infod6:lengthi10826029e4:name23:mediawiki-1.15.1.tar.gz12:piece%20lengthi10826029e6:pieces20:%bc%6f%a7%90%b7%73%88%92%c6%b4%15%fc%76%65%8a%97%67%63%71%5de8:url-listl68:http%3A%2F%2Fdownload.wikimedia.org%2Fmediawiki%2F1.15%2Fmediawiki-1.15.1.tar.gzee

*/

  function make_micro_torrent(file){ if (file.name && file.size && file.hash && file.hash.sha1){ var torrent = ["data:application/x-bittorrent;,d"]; if (file.trackers && file.trackers.length>0){ torrent.push("8:announce"); push_string(file.trackers[0], torrent); if (file.trackers.length>1){ torrent.push("13:announce-listl"); for (var i = 1; file.trackers.length>i; i++){ var tracker = encodeURIComponent(decodeURIComponent(file.trackers[i])); push_string(tracker, torrent); } torrent.push("e"); } } torrent.push("4:infod"); torrent.push("6:length"); torrent.push("i"); torrent.push(file.size); torrent.push("e"); torrent.push("4:name"); var name = encodeURIComponent(file.name); push_string(name, torrent); torrent.push("12:piece%20length"); torrent.push("i"); torrent.push(file.size < 16384 ? 16384 : file.size); torrent.push("e"); torrent.push("6:pieces"); var sha1 = file.hash.sha1; if (sha1.length < 40) sha1 = base32_to_hex(sha1); sha1 = sha1.replace(/[0-9A-Fa-f]{2}/g,"%$&"); push_string(sha1, torrent); torrent.push("e"); if (file.url && file.url.length>0){ torrent.push("8:url-listl"); for (var i = 0; file.url.length>i; i++){ var url = encodeURIComponent(decodeURIComponent(file.url[i])); push_string(url, torrent); } torrent.push("e"); } torrent.push("e"); return torrent.join(""); } } 

/*
Function to insert new links to the page.
*/

 function insert_link_after(sibling, href, class_name){ var new_link = document.createElement("A"); if (href) new_link.href = href; if (class_name) new_link.className = class_name; return sibling.parentNode.insertBefore(new_link, sibling.nextSibling); } 

/*

Go

We look through the links and look for magnets in them. Links are moved in the reverse order as we add new ones and the array of links increases automatically. Now I didn’t check it but I used to end up on an endless loop on it.
*/

  var links = document.getElementsByTagName("A"); for (var i = links.length - 1; i >= 0; i--){ var link = links[i]; var magnet_index = link.href.indexOf("magnet:?"); var magnet_link; if (magnet_index < 0){ magnet_index = link.href.indexOf("magnet%3A%3F"); if( magnet_index > -1 ) magnet_link = try_decode(link.href.substr(magnet_index)); } else magnet_link = link.href.substr(magnet_index); if (magnet_index > -1){ var file = parse_magnet(magnet_link); if ( file.name ){ var spliter = "/" + file.name + "#magnet"; if ( link.href.indexOf( spliter ) > -1 ){ /* URL + magnet     . */ if (!file.url) file.url = []; file.url.push(link.href.split( spliter )[0] + "/" + file.name); } } var link_functions = { "micro-torrent": make_micro_torrent, "torrent-magnet":torrent_magnet, "dc-magnet": dc_magnet, "ed2k-link": ed2k_link, "full-magnet": full_magnet}; for (var name in link_functions){ var href = link_functions[name](file); if (href){ var a = insert_link_after(link, href, name); if ( name == "micro-torrent" ){ /*   download       . */ a.setAttribute("download", (file.name) + ".micro.torrent"); } } } } } 

/*
Text links set through styles. You can also enter pictures in the form of a Data URI .
*/

 document.head.appendChild(document.createElement('style')).appendChild(document.createTextNode(` [class*='-magnet'], .ed2k-link, .micro-torrent{ margin-left: 1em } a[class*='-magnet']:before, .ed2k-link:before, .micro-torrent:before{ content: attr(class) }`)) })(); void(0); 

Installation

1. Copy the folder “magnet-converter” with the contents to a local disk.
2. In the Chrome browser, open “chrome: // extensions /” or “Main Menu> Additional Tools> Extensions”
3. Put a tick "Developer Mode"
4. Click "Download unpacked extension ..."
5. We find and select the folder “magnet-converter” with scripts.
6. Click OK.

Test

Link for the test script:

Test

The result should look like this:

full-magnet ed2k-link dc-magnet torrent-magnet micro-torrent
There will be 5 additional links for different p2p clients.

Conclusion

I hope that the developers of p2p clients will add a code so that the universal magnet is recognized by the client correctly and there is no need to divide it.

References:

Magnet Internet 1 - Magnet
Magnet Online 2 - Hypertext
Internet Magnets 3 - P2P Website and Forum
Internet on magnets 4 - Divide the magnet into pieces
Internet Magnets 5 - Lighthouses and Messages (Personal, Public, and Updates)