anomie / yt-user (public) (License: Unspecified) (since 2018-07-31) (hash sha1)
No description available
Clone URLs: https://rocketgit.com/user/anomie/yt-user ssh://rocketgit@ssh.rocketgit.com/user/anomie/yt-user git://git.rocketgit.com/user/anomie/yt-user
master
List of commits:
Subject Hash Author Date (UTC)
Fix offset bug 6031763c679edb69a5197a9e2544c3791c20a08e terorie 2018-07-30 01:16:59
Minimal async HTTP cf46dcf53f92355652046732b049d66f78170151 terorie 2018-07-30 01:09:23
Multithreading proof-of-concept 593c61acca56f9bebbdb8ded27d98c7c016d7d8e terorie 2018-07-30 01:04:14
Fix CLI for channel dumps e2ea0a179ea584fa994a05dc913298c904060412 terorie 2018-07-29 22:10:50
Move work command fec4a5e0677f2f72a87db79fd673aab17c776241 terorie 2018-07-29 21:56:21
Set API by CLI a9589194e7b5cbd33d5cf0fc96d71334d094520a terorie 2018-07-29 21:53:47
include GPLv3 e9b14041d4ecc991f1c40ecff4e13b737b1ccc15 terorie 2018-07-29 21:34:50
Refactor API selection c3b6047f98184e618ceb88965583e2aabe71fb7d terorie 2018-07-29 21:34:19
Channel url dump CLI 0429c2f1ceef84d2e7505591a0fee9babce7eb52 terorie 2018-07-29 02:09:59
Cobra & channel video dumper b6a0e418403b24df1162b749878d24a9918f9b8c terorie 2018-07-28 23:49:38
Rename project to yt-mango 0f19dcfd3f54c3902a0aa97b8c0f4aa932538d73 terorie 2018-07-28 15:10:49
Remove debug output 9f5b9dd87324b30044723539875ee8a419b7eee0 terorie 2018-07-28 14:45:13
Decode video formats bafc6c4ad3587e2de7a88cf532ad1df5ee5a7964 terorie 2018-07-28 14:34:57
Add formats from youtube-dl e612768d79ed1743b415073d9dba713e785e2ea6 terorie 2018-07-28 13:33:48
Initial README 0e9d46ecadcc5b019b9fae4e356bc3360f521b5e terorie 2018-07-28 03:56:26
Initial commit 3b5744c589494163e05a9d54fba10c71e5965d6a terorie 2018-07-28 03:55:53
Commit 6031763c679edb69a5197a9e2544c3791c20a08e - Fix offset bug
Refactor channel ID parsing
Author: terorie
Author date (UTC): 2018-07-30 01:16
Committer name: terorie
Committer date (UTC): 2018-07-30 01:16
Parent(s): cf46dcf53f92355652046732b049d66f78170151
Signing key:
Tree: 2ff59f79fe88b5a459b3e3f6229c30758c4a6191
File Lines added Lines deleted
api/ids.go 49 0
cmd/channel.go 0 3
cmd/channeldump.go 6 35
File api/ids.go added (mode: 100644) (index 0000000..a480764)
1 package api
2
3 import (
4 "regexp"
5 "os"
6 "strings"
7 "log"
8 "net/url"
9 )
10
11 var matchChannelID = regexp.MustCompile("^([\\w\\-]|(%3[dD]))+$")
12
13 func GetChannelID(chanURL string) (string, error) {
14 if !matchChannelID.MatchString(chanURL) {
15 // Check if youtube.com domain
16 _url, err := url.Parse(chanURL)
17 if err != nil || (_url.Host != "www.youtube.com" && _url.Host != "youtube.com") {
18 log.Fatal("Not a channel ID:", chanURL)
19 os.Exit(1)
20 }
21
22 // Check if old /user/ URL
23 if strings.HasPrefix(_url.Path, "/user/") {
24 // TODO Implement extraction of channel ID
25 log.Fatal("New /channel/ link is required!\n" +
26 "The old /user/ links do not work.")
27 os.Exit(1)
28 }
29
30 // Remove /channel/ path
31 channelID := strings.TrimPrefix(_url.Path, "/channel/")
32 if len(channelID) == len(_url.Path) {
33 // No such prefix to be removed
34 log.Fatal("Not a channel ID:", channelID)
35 os.Exit(1)
36 }
37
38 // Remove rest of path from channel ID
39 slashIndex := strings.IndexRune(channelID, '/')
40 if slashIndex != -1 {
41 channelID = channelID[:slashIndex]
42 }
43
44 return channelID, nil
45 } else {
46 // It's already a channel ID
47 return chanURL, nil
48 }
49 }
File cmd/channel.go changed (mode: 100644) (index 89ace23..cf2ea8b)
... ... package cmd
2 2
3 3 import ( import (
4 4 "github.com/spf13/cobra" "github.com/spf13/cobra"
5 "regexp"
6 5 ) )
7 6
8 7 var force bool var force bool
 
... ... var Channel = cobra.Command{
13 12 Short: "Get information about a channel", Short: "Get information about a channel",
14 13 } }
15 14
16 var matchChannelID = regexp.MustCompile("^([\\w\\-]|(%3[dD]))+$")
17
18 15 func init() { func init() {
19 16 channelDumpCmd.Flags().BoolVarP(&force, "force", "f", false, "Overwrite the output file if it already exists") channelDumpCmd.Flags().BoolVarP(&force, "force", "f", false, "Overwrite the output file if it already exists")
20 17 channelDumpCmd.Flags().UintVar(&offset, "page-offset", 1, "Start getting videos at this page. (A page is usually 30 videos)") channelDumpCmd.Flags().UintVar(&offset, "page-offset", 1, "Start getting videos at this page. (A page is usually 30 videos)")
File cmd/channeldump.go changed (mode: 100644) (index 1fb88ce..1bff985)
... ... package cmd
2 2
3 3 import ( import (
4 4 "github.com/spf13/cobra" "github.com/spf13/cobra"
5 "net/url"
6 5 "os" "os"
7 "strings"
8 6 "time" "time"
9 7 "bufio" "bufio"
10 8 "log" "log"
 
... ... var channelDumpCmd = cobra.Command{
39 37 } }
40 38 channelDumpContext.printResults = printResults channelDumpContext.printResults = printResults
41 39
42 if !matchChannelID.MatchString(channelID) {
43 // Check if youtube.com domain
44 _url, err := url.Parse(channelID)
45 if err != nil || (_url.Host != "www.youtube.com" && _url.Host != "youtube.com") {
46 log.Fatal("Not a channel ID:", channelID)
47 os.Exit(1)
48 }
49
50 // Check if old /user/ URL
51 if strings.HasPrefix(_url.Path, "/user/") {
52 // TODO Implement extraction of channel ID
53 log.Fatal("New /channel/ link is required!\n" +
54 "The old /user/ links do not work.")
55 os.Exit(1)
56 }
57
58 // Remove /channel/ path
59 channelID = strings.TrimPrefix(_url.Path, "/channel/")
60 if len(channelID) == len(_url.Path) {
61 // No such prefix to be removed
62 log.Fatal("Not a channel ID:", channelID)
63 os.Exit(1)
64 }
65
66 // Remove rest of path from channel ID
67 slashIndex := strings.IndexRune(channelID, '/')
68 if slashIndex != -1 {
69 channelID = channelID[:slashIndex]
70 }
40 channelID, err := api.GetChannelID(channelID)
41 if err != nil {
42 log.Print(err)
43 os.Exit(1)
71 44 } }
72 45
73 46 log.Printf("Starting work on channel ID \"%s\".", channelID) log.Printf("Starting work on channel ID \"%s\".", channelID)
 
... ... var channelDumpCmd = cobra.Command{
115 88 page++ page++
116 89 } }
117 90 terminate: terminate:
118 log.Printf("&")
119 91
120 92 // Requests sent, wait for remaining requests to finish // Requests sent, wait for remaining requests to finish
121 93 for { for {
122 done := atomic.LoadUint64(&channelDumpContext.pagesDone)
123 // Page starts at 1
124 target := uint64(page) - 1
94 done := uint64(offset) + atomic.LoadUint64(&channelDumpContext.pagesDone)
95 target := uint64(page)
125 96 if done >= target { break } if done >= target { break }
126 97
127 98 // TODO use semaphore // TODO use semaphore
Hints:
Before first commit, do not forget to setup your git environment:
git config --global user.name "your_name_here"
git config --global user.email "your@email_here"

Clone this repository using HTTP(S):
git clone https://rocketgit.com/user/anomie/yt-user

Clone this repository using ssh (do not forget to upload a key first):
git clone ssh://rocketgit@ssh.rocketgit.com/user/anomie/yt-user

Clone this repository using git:
git clone git://git.rocketgit.com/user/anomie/yt-user

You are allowed to anonymously push to this repository.
This means that your pushed commits will automatically be transformed into a merge request:
... clone the repository ...
... make some changes and some commits ...
git push origin main