Skip to content

Instantly share code, notes, and snippets.

@TransparentLC
Last active August 19, 2023 02:29
Show Gist options
  • Star 16 You must be signed in to star a gist
  • Fork 4 You must be signed in to fork a gist
  • Save TransparentLC/72b6ea086b9a28964496c2bc1b721aa4 to your computer and use it in GitHub Desktop.
Save TransparentLC/72b6ea086b9a28964496c2bc1b721aa4 to your computer and use it in GitHub Desktop.
#!/usr/bin/env pwsh
# https://stackoverflow.com/questions/8761888/capturing-standard-out-and-error-with-start-process
function Start-Command ([String]$Path, [String]$Arguments) {
$pinfo = New-Object System.Diagnostics.ProcessStartInfo
$pinfo.FileName = $Path
$pinfo.RedirectStandardError = $true
$pinfo.RedirectStandardOutput = $true
$pinfo.UseShellExecute = $false
$pinfo.Arguments = $Arguments
$p = New-Object System.Diagnostics.Process
$p.StartInfo = $pinfo
$p.Start() | Out-Null
$p.WaitForExit()
return @{
stdout = $p.StandardOutput.ReadToEnd()
stderr = $p.StandardError.ReadToEnd()
ExitCode = $p.ExitCode
}
}
# https://stackoverflow.com/questions/1783554/fast-and-simple-binary-concatenate-files-in-powershell
function Join-File ([String[]]$Path, [String]$Destination) {
$OutFile = [IO.File]::Create($Destination)
foreach ($File in $Path) {
$InFile = [IO.File]::OpenRead($File)
$InFile.CopyTo($OutFile)
$InFile.Dispose()
}
$OutFile.Dispose()
}
function Get-RandomBytes ([Int32]$Length) {
$RNG = [Security.Cryptography.RandomNumberGenerator]::Create()
$Bytes = [Byte[]]::new($Length)
$RNG.GetBytes($Bytes)
return $Bytes
}
function Invoke-EncryptFile ([String]$Path, [Byte[]]$Key, [Byte[]]$IV, [String]$OutFile) {
$File = [IO.File]::ReadAllBytes($Path)
$AES = [Security.Cryptography.Aes]::Create()
$AES.Key = $Key
$AES.IV = $IV
$Encryptor = $AES.CreateEncryptor($AES.Key, $AES.IV)
$Encrypted = $Encryptor.TransformFinalBlock($File, 0, $File.Length)
[IO.File]::WriteAllBytes($OutFile, $Encrypted)
}
# $FFmpegExec = 'D:\ffmpeg\bin\ffmpeg.exe'
# $CurlExec = 'D:\curl-7.68.0-win64-mingw\bin\curl.exe'
$FFmpegExec = 'ffmpeg'
$CurlExec = 'curl'
if (Test-Path alias:\curl) {
Remove-Item alias:\curl
}
$Video = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath((Read-Host 'Input H264+AAC video path').Replace('"', ''))
$TempDirectory = [IO.Path]::GetDirectoryName($Video) + '/' + [GUID]::NewGuid().ToString('N')
New-Item $TempDirectory -ItemType Directory
&$FFmpegExec -i $Video -c copy -vbsf h264_mp4toannexb -absf aac_adtstoasc ($TempDirectory + '/video.ts')
&$FFmpegExec -i ($TempDirectory + '/video.ts') -c copy -f segment -segment_list ($TempDirectory + '/video.m3u8') ($TempDirectory + '/%d.ts')
Remove-Item ($TempDirectory + '/video.ts')
$tsCount = (Get-ChildItem $TempDirectory -Filter *.ts).Length
do {
$LimitExceed = $false
foreach ($ts in (Get-ChildItem $TempDirectory -Filter *.ts)) {
if ($ts.Length -gt 5MB) {
$LimitExceed = $true
Write-Host '[ERROR]' -NoNewline -BackgroundColor DarkRed -ForegroundColor White
Write-Host (' File size limit exceeded: {0} ({1} MB)' -f $ts.Name, [Math]::Round($ts.Length / 1MB, 2))
}
}
if ($LimitExceed) {
Read-Host 'Compress the files and press enter to continue'
}
} while ($LimitExceed)
Write-Host 'Parsing M3U8...'
$m3u8Source = [IO.File]::ReadAllLines($TempDirectory + '/video.m3u8')
$m3u8 = @{
'meta' = @();
'info' = @();
}
for ($i = 0; $i -lt $m3u8Source.Count; $i++) {
if (($m3u8Source[$i] -eq '#EXTM3U') -or ($m3u8Source[$i] -eq '#EXT-X-ENDLIST')) {
continue
}
if ($m3u8Source[$i].StartsWith('#EXTINF:')) {
$m3u8.info += @{
'duration' = [Double]($m3u8Source[$i].Replace('#EXTINF:', '').Replace(',', ''));
'file' = $m3u8Source[++$i];
}
} else {
$m3u8.meta += $m3u8Source[$i]
}
}
Write-Host 'Merging TS files...'
$FastStart = 3
for ($i = 0; $i -lt $tsCount; $i++) {
$LastPath = ('{0}/{1}.ts' -f $TempDirectory, ($i - 1))
$CurrentPath = ('{0}/{1}.ts' -f $TempDirectory, $i)
if (-not [IO.File]::Exists($LastPath)) {
continue
}
if ((Get-Item $LastPath).Length + (Get-Item $CurrentPath).Length -le @(2MB, 1MB)[[Bool]$FastStart]) {
Write-Host '[MERGE]' -NoNewline -BackgroundColor DarkGreen -ForegroundColor White
Write-Host (' {0}.ts <- {1}.ts' -f $i, ($i - 1))
Join-File -Path $LastPath, $CurrentPath -Destination ('{0}/~.ts' -f $TempDirectory)
Remove-Item $LastPath
Remove-Item $CurrentPath
Rename-Item ('{0}/~.ts' -f $TempDirectory) ('{0}.ts' -f $i)
} else {
Write-Host '[SKIP]' -NoNewline -BackgroundColor DarkCyan -ForegroundColor White
if ($FastStart -gt 0) {
$FastStart--
Write-Host (' {0}.ts ({1} MB FastStart)' -f $i, [Math]::Round((Get-Item $LastPath).Length / 1MB, 2))
} else {
Write-Host (' {0}.ts ({1} MB)' -f $i, [Math]::Round((Get-Item $LastPath).Length / 1MB, 2))
}
}
}
Write-Host 'Writing M3U8...'
$MergedInfo = @()
$tsLast = 0
for ($i = 0; $i -lt $tsCount; $i++) {
if (-not [IO.File]::Exists(('{0}/{1}.ts' -f $TempDirectory, $i))) {
continue
}
$MergedDuration = 0
for ($j = $tsLast; $j -le $i; $j++) {
$MergedDuration += $m3u8.info[$j].duration
}
$MergedInfo += @{
'duration' = $MergedDuration;
'file' = $m3u8.info[$i].file
}
$tsLast = $i + 1
}
$m3u8.info = $MergedInfo
$m3u8Content = @('#EXTM3U')
foreach ($meta in $m3u8.meta) {
$m3u8Content += $meta
}
foreach ($info in $m3u8.info) {
$m3u8Content += '#EXTINF:' + $info.duration + ','
$m3u8Content += $info.file
}
$m3u8Content += '#EXT-X-ENDLIST'
[IO.File]::WriteAllLines($TempDirectory + '/video.m3u8', $m3u8Content)
Read-Host 'Press enter to start uploading'
$EncryptKey = Get-RandomBytes 16
$EncryptIV = Get-RandomBytes 16
Write-Host ('Key: ' + ($EncryptKey | ForEach-Object { '{0:X2}' -f $_ }))
Write-Host ('IV: ' + ($EncryptIV | ForEach-Object { '{0:X2}' -f $_ }))
Write-Host 'Uploading Key File...'
[IO.File]::WriteAllBytes($TempDirectory + '/KEY', $EncryptKey)
$Response = (Start-Command $CurlExec (@(
'https://kfupload.alibaba.com/mupload',
'-H "User-Agent: iAliexpress/8.27.0 (iPhone; iOS 12.1.2; Scale/2.00)"',
'-X POST',
'-F scene=productImageRule',
'-F name=image.jpg',
'-F file=@{0}' -f ($TempDirectory + '/KEY')
) -join ' ')).stdout | ConvertFrom-Json
$EncryptKeyURL = $Response.url
Remove-Item ($TempDirectory + '/KEY')
$m3u8.meta += ('#EXT-X-KEY:METHOD=AES-128,URI="{0}",IV=0x{1}' -f $EncryptKeyURL,(($EncryptIV | ForEach-Object { '{0:x2}' -f $_ }) -join ''))
$URLMapping = @{}
foreach ($ts in (Get-ChildItem $TempDirectory -Filter *.ts)) {
$FileName = [IO.Path]::GetFileName($ts.FullName)
$FileEncrypted = $ts.FullName + '.encrypt'
Invoke-EncryptFile $ts.FullName $EncryptKey $EncryptIV $FileEncrypted
$Response = (Start-Command $CurlExec (@(
'https://kfupload.alibaba.com/mupload',
'-H "User-Agent: iAliexpress/8.27.0 (iPhone; iOS 12.1.2; Scale/2.00)"',
'-X POST',
'-F scene=productImageRule',
'-F name=image.jpg',
'-F file=@{0}' -f $FileEncrypted
) -join ' ')).stdout | ConvertFrom-Json
Remove-Item $FileEncrypted
if ([Bool][Int32]$Response.code) {
Write-Host '[WARNING]' -NoNewline -BackgroundColor DarkYellow -ForegroundColor White
Write-Host (' {0} Failed to upload' -f $FileName)
$URL = $FileName
} else {
Write-Host '[UPLOAD]' -NoNewline -BackgroundColor DarkGreen -ForegroundColor White
Write-Host (' {0} {1}' -f $FileName, $Response.url)
$URL = $Response.url
}
$URLMapping.$FileName = $URL
}
Write-Host 'Writing M3U8 with URL...'
$m3u8Content = @('#EXTM3U')
foreach ($meta in $m3u8.meta) {
$m3u8Content += $meta
}
foreach ($info in $m3u8.info) {
$m3u8Content += '#EXTINF:' + $info.duration + ','
$m3u8Content += $URLMapping.($info.file)
}
$m3u8Content += '#EXT-X-ENDLIST'
[IO.File]::WriteAllLines($TempDirectory + '/video_online.m3u8', $m3u8Content)
Write-Host 'Complete!'
@lizkes
Copy link

lizkes commented Aug 14, 2021

使用 Python 的 requests模块上传的食用方法是这样:

  response = requests.post(
      url="https://kfupload.alibaba.com/mupload",
      headers={
          "user-agent": "iAliexpress/8.27.0 (iPhone; iOS 12.1.2; Scale/2.00)",
      },
       # path是pathlib模块里的Path类
      files={"file": (path.name, path.open("rb"), "image/jpeg")},
      data={"name": "image.jpg", "scene": "productImageRule"},
  )

实测可行

@lizkes
Copy link

lizkes commented Aug 14, 2021

小小的魔改了一下,主要是改动了ffmpeg的指令,重新编码视频流来插入cgop关键帧,从而减小ts文件的大小,应该再也不会大于5MB了...
缺点是因为要重新转码,所以需要花费一定时间。
这里是改动后的版本

@TransparentLC
Copy link
Author

小小的魔改了一下,主要是改动了ffmpeg的指令,重新编码视频流来插入cgop关键帧,从而减小ts文件的大小,应该再也不会大于5MB了...
缺点是因为要重新转码,所以需要花费一定时间。
这里是改动后的版本

倒也不用在脚本里全部二压一遍 23333

我当时的想法是脚本只负责切片然后把所有超过 5 MB 的 ts 在上传前单独列出来自行压缩后再继续(或许能改成在脚本里自动运行?),而上传前压制视频的时候可以使用各种参数(比如 keyint)控制 GOP 大小。

虽然上传的文件大小上限是 5 MB 不过合并的时候我设定是达到 2 MB 就不合并了,ts 太大的话开始播放和跳转的时候加载第一个要等很久。

@ludashi2020
Copy link

尝试一下,你改的这个脚本运行报错。`-rw-r--r-- 1 root root 1532 Nov 8 2020 ok_bbrplus_
centos.sh.1
-rw-r--r--. 1 root root 18513 Dec 21 2018 package-loc
k.json
-rw-r--r-- 1 root root 6672 May 1 2020 pgdg-redhat
-repo-latest.noarch.rpm
-rw-r--r-- 1 root root 41261748 May 25 2017 powershell-
6.0.0_alpha.14-1.el7.centos.x86_64.rpm
drwxr-xr-x. 17 501 501 4096 Dec 24 2018 Python-3.6.
2
-rw-r--r--. 1 root root 16907204 Jul 17 2017 Python-3.6.
2.tar.xz
drwxr-xr-x 2 root root 4096 Mar 29 2018 Resilio-Syn
c-master
-rw-r--r-- 1 root root 3592 Oct 31 2018 rskernel.sh
-rw-r--r-- 1 root root 22177073 Nov 5 2020 sphinx-3.3.
1-b72d67b-linux-amd64.tar.gz
-rw-r--r--. 1 root root 2736 Dec 25 2018 th2.py
-rw-r--r--. 1 root root 2736 Dec 25 2018 th3.py
-rw-r--r--. 1 root root 2736 Dec 25 2018 th4.py
-rw-r--r-- 1 root root 25706 Nov 5 2020 thankyou.ht
ml
-rw-r--r--. 1 root root 2736 Dec 24 2018 th.py
-rwxr-xr-x 1 root root 171 Feb 28 2019 yuesehan.sh
[root@yesehan ~]# cd /home
[root@yesehan home]# ll
total 77025516
-rw-r--r-- 1 root root 20323 Oct 12 2019 1.7z
-rw-r--r-- 1 root root 3729632 Mar 10 16:23 6c3b671
5-d30d-4513-bba5-9e9688220063.mp4
-rwxrwxrwx 1 root root 8605 Aug 15 10:19 alicdn.
ps1
drwxr-xr-x. 3 root root 4096 Jul 8 2019 bak
-rwxr-xr-x 1 root root 14127 Oct 9 2019 BDW.sh
drwxr-xr-x 3 root root 4096 Feb 29 2020 ccaa-ma
ster
drwxr-xr-x. 8 root root 4096 Aug 23 2019 ckspide
r
-rw-r--r-- 1 root root 1468921 Sep 27 2019 ckspide
r.zip
drwxr-xr-x 2 root root 4096 Oct 12 2019 dht
-rw-r--r-- 1 root root 10136873404 Jun 20 2020 dhtdb_a
ll_backup.sql
drwxr-xr-x 4 root root 4096 Oct 10 2019 dhtdbs
-rw-r--r-- 1 root root 6153 Aug 15 10:09 f.mp4
drwxr-xr-x 8 root root 4096 Feb 19 16:13 free-hl
s
drwxr-xr-x 16 root mariadb 4096 Dec 2 2020 mariadb
-rw-r--r-- 1 root root 574358 Apr 7 2020 master.
zip
drwxr-xr-x. 16 root mysql 4096 Jun 20 2020 mysql_b
ak
-rw------- 1 root root 72549 Oct 9 2019 nohup.o
ut
drwxr-xr-x 4 root root 4096 Mar 29 17:06 PicVid
drwxr-xr-x 3 root root 4096 Oct 10 2019 Resilio
-Sync-master
drwxr-xr-x. 3 root root 4096 Oct 10 2019 RslSync
-rw-r--r-- 1 root root 11573813 Dec 18 2017 sphinx-
3.0.1-7fec4f6-linux-amd64.tar.gz
drwxr-xr-x 22 root root 4096 Oct 11 2019 sphinx-
jieba
-rw-r--r-- 1 root root 68719476736 Apr 6 2020 swapfil
e
drwxrwxrwx 3 www www 208896 Aug 15 09:32 tmp
drwx------. 4 www www 4096 Apr 7 2020 www
drwxrwxrwx. 2 root root 4096 Jul 30 10:18 wwwlogs
drwxr-xr-x. 29 root root 4096 Jul 30 10:37 wwwroot
[root@yesehan home]# ./alicdn.ps1

Directory: /home

Mode LastWriteTime Length Name


d---- 8/15/2021 10:51 AM 9fee8975a
b2e47adb9
e1d2381ff
4fa8e
ffmpeg version N-58069-gc253b180cb-static https://johnvansic
kle.com/ffmpeg/ Copyright (c) 2000-2021 the FFmpeg develope
rs
built with gcc 8 (Debian 8.3.0-6)
configuration: --enable-gpl --enable-version3 --enable-sta
tic --disable-debug --disable-ffplay --disable-indev=sndio -
-disable-outdev=sndio --cc=gcc --enable-fontconfig --enable-
frei0r --enable-gnutls --enable-gmp --enable-libgme --enable
-gray --enable-libaom --enable-libfribidi --enable-libass --
enable-libvmaf --enable-libfreetype --enable-libmp3lame --en
able-libopencore-amrnb --enable-libopencore-amrwb --enable-l
ibopenjpeg --enable-librubberband --enable-libsoxr --enable-
libspeex --enable-libsrt --enable-libvorbis --enable-libopus
--enable-libtheora --enable-libvidstab --enable-libvo-amrwb
enc --enable-libvpx --enable-libwebp --enable-libx264 --enab
le-libx265 --enable-libxml2 --enable-libdav1d --enable-libxv
id --enable-libzvbi --enable-libzimg
libavutil 57. 2.100 / 57. 2.100
libavcodec 59. 3.102 / 59. 3.102
libavformat 59. 4.101 / 59. 4.101
libavdevice 59. 0.100 / 59. 0.100
libavfilter 8. 0.103 / 8. 0.103
libswscale 6. 0.100 / 6. 0.100
libswresample 4. 0.100 / 4. 0.100
libpostproc 56. 0.100 / 56. 0.100
-c:v: Protocol not found
Did you mean file:-c:v?
Parsing M3U8...
MethodInvocationException: /home/alicdn.ps1:80
Line |
80 | $m3u8Source = [IO.File]::ReadAllLines($TempDirectory

  • '/video.m3u8')
    | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     | Exception calling "ReadAllLines" with "1"
     | argument(s): "Could not find file
     | '/home/9fee8975ab2e47adb9e1d2381ff4fa8e/video.m3u8'."

Merging TS files...
Writing M3U8...
Press enter to start uploading:
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
������������������������������������������������������������
'

@ludashi2020
Copy link

执行脚本之后报错,原本的不会。但是原本上传不了视频。你的报错,不知能不能上传。` libpostproc 56. 0.100 / 56. 0.100
-c:v: Protocol not found
Did you mean file:-c:v?
Parsing M3U8...
MethodInvocationException: /home/alicdn.ps1:80
Line |
80 | $m3u8Source = [IO.File]::ReadAllLines($TempDirectory

  • '/video.m3u8')
    | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     | Exception calling "ReadAllLines" with "1"
     | argument(s): "Could not find file
     | '/home/9fee8975ab2e47adb9e1d2381ff4fa8e/video.m3u8'."

Merging TS files...
Writing M3U8...
Press enter to start uploading:`

@lizkes
Copy link

lizkes commented Aug 15, 2021

@TransparentLC

我当时的想法是脚本只负责切片然后把所有超过 5 MB 的 ts 在上传前单独列出来自行压缩后再继续(或许能改成在脚本里自动运行?),而上传前压制视频的时候可以使用各种参数(比如 keyint)控制 GOP 大小。

又研究了一下,发现根本没必要进行二次压缩,只需要在ffmpeg参数里添加

-hls_time 1 -hls_flags split_by_time

ffmpeg就会以每秒一个ts文件的标准来切片,正常视频不可能超过5MB...
我的代码

@zsbai
Copy link

zsbai commented Aug 15, 2021

使用 Python 的 requests模块上传的食用方法是这样:

  response = requests.post(
      url="https://kfupload.alibaba.com/mupload",
      headers={
          "user-agent": "iAliexpress/8.27.0 (iPhone; iOS 12.1.2; Scale/2.00)",
      },
       # path是pathlib模块里的Path类
      files={"file": (path.name, path.open("rb"), "image/jpeg")},
      data={"name": "image.jpg", "scene": "productImageRule"},
  )

实测可行

你好,使用你的代码,返回为{'code': '1'},然后文件路径用的是open(path,'rb')这样子,照片是300kb的png照片...没找到哪里出问题了

 response = requests.post(
...       url="https://kfupload.alibaba.com/mupload",
...       headers={
...           "user-agent": "iAliexpress/8.27.0 (iPhone; iOS 12.1.2; Scale/2.00)",
...       },
...        # path是pathlib模块里的Path类
...       files={"file": (open('/Users/xxx/2.png','rb'))},
...       data={"name": "image.png", "scene": "productImageRule"},
...   )

@lizkes
Copy link

lizkes commented Aug 15, 2021

@zsbai
应该是文件路径填错了,或者没有转义,仔细检查

@zsbai
Copy link

zsbai commented Aug 16, 2021

@zsbai
应该是文件路径填错了,或者没有转义,仔细检查

试了好多次...路径检查没问题,open 可以直接在预览打开图片
截屏2021-08-16 下午2 07 54

并且测试了一下其他的图床,是可以正常上传的
截屏2021-08-16 下午2 11 58

@lizkes
Copy link

lizkes commented Aug 16, 2021

摊手,我这边没啥问题
完整代码

@zsbai
Copy link

zsbai commented Aug 16, 2021

摊手,我这边没啥问题
完整代码

我的天,找到问题,我没有传入文件名,修改成下面这样成功了:

r = requests.post(url = url, headers = header, data = data , files = {'file':('file',open('/Users/bailu/2.png','rb'),'image/jpeg')})

不好意思,粗心了

@ludashi2020
Copy link

是不是上传接口变了?
`ConvertFrom-Json: /home/2.ps1:198
Line |
198 | ) -join ' ')).stdout | ConvertFrom-Json
| ~~~~~~~~~~~~~~~~
| Conversion from JSON failed with error: Unexpected character encountered while parsing value: <. Path '', line 0, position 0.

[UPLOAD] 0.ts
ConvertFrom-Json: /home/2.ps1:198
Line |
198 | ) -join ' ')).stdout | ConvertFrom-Json
| ~~~~~~~~~~~~~~~~
| Conversion from JSON failed with error: Unexpected character encountered while parsing value: <. Path '', line 0, position 0.

[UPLOAD] 1.ts
ConvertFrom-Json: /home/2.ps1:198
Line |
198 | ) -join ' ')).stdout | ConvertFrom-Json
| ~~~~~~~~~~~~~~~~
| Conversion from JSON failed with error: Unexpected character encountered while parsing value: <. Path '', line 0, position 0.

[UPLOAD] 14.ts

`

@1265578519
Copy link

2022-02-11检测正常

{
  "fs_url": "Hf5f5ff99f65041fdb897334abddf63fdP.jpg",
  "code": "0",
  "size": "19674",
  "width": "533",
  "url": "https://ae01.alicdn.com/kf/Hf5f5ff99f65041fdb897334abddf63fdP.jpg",
  "hash": "aa9de6dcaf63826d3f6b27769883f5e1",
  "height": "147"
}

@lizkes
Copy link

lizkes commented Apr 25, 2022

不能用了...显示502 bad gateway

@ludashi2020
Copy link

上传接口又改了,求最新接口

@bebestmaple
Copy link

上传接口又改了,求最新接口

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment