flv-demuxer.js 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101
  1. /*
  2. * Copyright (C) 2016 Bilibili. All Rights Reserved.
  3. *
  4. * @author zheng qian <xqq@xqq.im>
  5. *
  6. * Licensed under the Apache License, Version 2.0 (the "License");
  7. * you may not use this file except in compliance with the License.
  8. * You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. import Log from '../utils/logger.js';
  19. import AMF from './amf-parser.js';
  20. import SPSParser from './sps-parser.js';
  21. import DemuxErrors from './demux-errors.js';
  22. import MediaInfo from '../core/media-info.js';
  23. import {IllegalStateException} from '../utils/exception.js';
  24. function Swap16(src) {
  25. return (((src >>> 8) & 0xFF) |
  26. ((src & 0xFF) << 8));
  27. }
  28. function Swap32(src) {
  29. return (((src & 0xFF000000) >>> 24) |
  30. ((src & 0x00FF0000) >>> 8) |
  31. ((src & 0x0000FF00) << 8) |
  32. ((src & 0x000000FF) << 24));
  33. }
  34. function ReadBig32(array, index) {
  35. return ((array[index] << 24) |
  36. (array[index + 1] << 16) |
  37. (array[index + 2] << 8) |
  38. (array[index + 3]));
  39. }
  40. class FLVDemuxer {
  41. constructor(probeData, config) {
  42. this.TAG = 'FLVDemuxer';
  43. this._config = config;
  44. this._onError = null;
  45. this._onMediaInfo = null;
  46. this._onMetaDataArrived = null;
  47. this._onScriptDataArrived = null;
  48. this._onTrackMetadata = null;
  49. this._onDataAvailable = null;
  50. this._dataOffset = probeData.dataOffset;
  51. this._firstParse = true;
  52. this._dispatch = false;
  53. this._hasAudio = probeData.hasAudioTrack;
  54. this._hasVideo = probeData.hasVideoTrack;
  55. this._hasAudioFlagOverrided = false;
  56. this._hasVideoFlagOverrided = false;
  57. this._audioInitialMetadataDispatched = false;
  58. this._videoInitialMetadataDispatched = false;
  59. this._mediaInfo = new MediaInfo();
  60. this._mediaInfo.hasAudio = this._hasAudio;
  61. this._mediaInfo.hasVideo = this._hasVideo;
  62. this._metadata = null;
  63. this._audioMetadata = null;
  64. this._videoMetadata = null;
  65. this._naluLengthSize = 4;
  66. this._timestampBase = 0; // int32, in milliseconds
  67. this._timescale = 1000;
  68. this._duration = 0; // int32, in milliseconds
  69. this._durationOverrided = false;
  70. this._referenceFrameRate = {
  71. fixed: true,
  72. fps: 23.976,
  73. fps_num: 23976,
  74. fps_den: 1000
  75. };
  76. this._flvSoundRateTable = [5500, 11025, 22050, 44100, 48000];
  77. this._mpegSamplingRates = [
  78. 96000, 88200, 64000, 48000, 44100, 32000,
  79. 24000, 22050, 16000, 12000, 11025, 8000, 7350
  80. ];
  81. this._mpegAudioV10SampleRateTable = [44100, 48000, 32000, 0];
  82. this._mpegAudioV20SampleRateTable = [22050, 24000, 16000, 0];
  83. this._mpegAudioV25SampleRateTable = [11025, 12000, 8000, 0];
  84. this._mpegAudioL1BitRateTable = [0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448, -1];
  85. this._mpegAudioL2BitRateTable = [0, 32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, -1];
  86. this._mpegAudioL3BitRateTable = [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, -1];
  87. this._videoTrack = {type: 'video', id: 1, sequenceNumber: 0, samples: [], length: 0};
  88. this._audioTrack = {type: 'audio', id: 2, sequenceNumber: 0, samples: [], length: 0};
  89. this._littleEndian = (function () {
  90. let buf = new ArrayBuffer(2);
  91. (new DataView(buf)).setInt16(0, 256, true); // little-endian write
  92. return (new Int16Array(buf))[0] === 256; // platform-spec read, if equal then LE
  93. })();
  94. }
  95. destroy() {
  96. this._mediaInfo = null;
  97. this._metadata = null;
  98. this._audioMetadata = null;
  99. this._videoMetadata = null;
  100. this._videoTrack = null;
  101. this._audioTrack = null;
  102. this._onError = null;
  103. this._onMediaInfo = null;
  104. this._onMetaDataArrived = null;
  105. this._onScriptDataArrived = null;
  106. this._onTrackMetadata = null;
  107. this._onDataAvailable = null;
  108. }
  109. static probe(buffer) {
  110. let data = new Uint8Array(buffer);
  111. let mismatch = {match: false};
  112. if (data[0] !== 0x46 || data[1] !== 0x4C || data[2] !== 0x56 || data[3] !== 0x01) {
  113. return mismatch;
  114. }
  115. let hasAudio = ((data[4] & 4) >>> 2) !== 0;
  116. let hasVideo = (data[4] & 1) !== 0;
  117. let offset = ReadBig32(data, 5);
  118. if (offset < 9) {
  119. return mismatch;
  120. }
  121. return {
  122. match: true,
  123. consumed: offset,
  124. dataOffset: offset,
  125. hasAudioTrack: hasAudio,
  126. hasVideoTrack: hasVideo
  127. };
  128. }
  129. bindDataSource(loader) {
  130. loader.onDataArrival = this.parseChunks.bind(this);
  131. return this;
  132. }
  133. // prototype: function(type: string, metadata: any): void
  134. get onTrackMetadata() {
  135. return this._onTrackMetadata;
  136. }
  137. set onTrackMetadata(callback) {
  138. this._onTrackMetadata = callback;
  139. }
  140. // prototype: function(mediaInfo: MediaInfo): void
  141. get onMediaInfo() {
  142. return this._onMediaInfo;
  143. }
  144. set onMediaInfo(callback) {
  145. this._onMediaInfo = callback;
  146. }
  147. get onMetaDataArrived() {
  148. return this._onMetaDataArrived;
  149. }
  150. set onMetaDataArrived(callback) {
  151. this._onMetaDataArrived = callback;
  152. }
  153. get onScriptDataArrived() {
  154. return this._onScriptDataArrived;
  155. }
  156. set onScriptDataArrived(callback) {
  157. this._onScriptDataArrived = callback;
  158. }
  159. // prototype: function(type: number, info: string): void
  160. get onError() {
  161. return this._onError;
  162. }
  163. set onError(callback) {
  164. this._onError = callback;
  165. }
  166. // prototype: function(videoTrack: any, audioTrack: any): void
  167. get onDataAvailable() {
  168. return this._onDataAvailable;
  169. }
  170. set onDataAvailable(callback) {
  171. this._onDataAvailable = callback;
  172. }
  173. // timestamp base for output samples, must be in milliseconds
  174. get timestampBase() {
  175. return this._timestampBase;
  176. }
  177. set timestampBase(base) {
  178. this._timestampBase = base;
  179. }
  180. get overridedDuration() {
  181. return this._duration;
  182. }
  183. // Force-override media duration. Must be in milliseconds, int32
  184. set overridedDuration(duration) {
  185. this._durationOverrided = true;
  186. this._duration = duration;
  187. this._mediaInfo.duration = duration;
  188. }
  189. // Force-override audio track present flag, boolean
  190. set overridedHasAudio(hasAudio) {
  191. this._hasAudioFlagOverrided = true;
  192. this._hasAudio = hasAudio;
  193. this._mediaInfo.hasAudio = hasAudio;
  194. }
  195. // Force-override video track present flag, boolean
  196. set overridedHasVideo(hasVideo) {
  197. this._hasVideoFlagOverrided = true;
  198. this._hasVideo = hasVideo;
  199. this._mediaInfo.hasVideo = hasVideo;
  200. }
  201. resetMediaInfo() {
  202. this._mediaInfo = new MediaInfo();
  203. }
  204. _isInitialMetadataDispatched() {
  205. if (this._hasAudio && this._hasVideo) { // both audio & video
  206. return this._audioInitialMetadataDispatched && this._videoInitialMetadataDispatched;
  207. }
  208. if (this._hasAudio && !this._hasVideo) { // audio only
  209. return this._audioInitialMetadataDispatched;
  210. }
  211. if (!this._hasAudio && this._hasVideo) { // video only
  212. return this._videoInitialMetadataDispatched;
  213. }
  214. return false;
  215. }
  216. // function parseChunks(chunk: ArrayBuffer, byteStart: number): number;
  217. parseChunks(chunk, byteStart) {
  218. if (!this._onError || !this._onMediaInfo || !this._onTrackMetadata || !this._onDataAvailable) {
  219. throw new IllegalStateException('Flv: onError & onMediaInfo & onTrackMetadata & onDataAvailable callback must be specified');
  220. }
  221. let offset = 0;
  222. let le = this._littleEndian;
  223. if (byteStart === 0) { // buffer with FLV header
  224. if (chunk.byteLength > 13) {
  225. let probeData = FLVDemuxer.probe(chunk);
  226. offset = probeData.dataOffset;
  227. } else {
  228. return 0;
  229. }
  230. }
  231. if (this._firstParse) { // handle PreviousTagSize0 before Tag1
  232. this._firstParse = false;
  233. if (byteStart + offset !== this._dataOffset) {
  234. Log.w(this.TAG, 'First time parsing but chunk byteStart invalid!');
  235. }
  236. let v = new DataView(chunk, offset);
  237. let prevTagSize0 = v.getUint32(0, !le);
  238. if (prevTagSize0 !== 0) {
  239. Log.w(this.TAG, 'PrevTagSize0 !== 0 !!!');
  240. }
  241. offset += 4;
  242. }
  243. while (offset < chunk.byteLength) {
  244. this._dispatch = true;
  245. let v = new DataView(chunk, offset);
  246. if (offset + 11 + 4 > chunk.byteLength) {
  247. // data not enough for parsing an flv tag
  248. break;
  249. }
  250. let tagType = v.getUint8(0);
  251. let dataSize = v.getUint32(0, !le) & 0x00FFFFFF;
  252. if (offset + 11 + dataSize + 4 > chunk.byteLength) {
  253. // data not enough for parsing actual data body
  254. break;
  255. }
  256. if (tagType !== 8 && tagType !== 9 && tagType !== 18) {
  257. Log.w(this.TAG, `Unsupported tag type ${tagType}, skipped`);
  258. // consume the whole tag (skip it)
  259. offset += 11 + dataSize + 4;
  260. continue;
  261. }
  262. let ts2 = v.getUint8(4);
  263. let ts1 = v.getUint8(5);
  264. let ts0 = v.getUint8(6);
  265. let ts3 = v.getUint8(7);
  266. let timestamp = ts0 | (ts1 << 8) | (ts2 << 16) | (ts3 << 24);
  267. let streamId = v.getUint32(7, !le) & 0x00FFFFFF;
  268. if (streamId !== 0) {
  269. Log.w(this.TAG, 'Meet tag which has StreamID != 0!');
  270. }
  271. let dataOffset = offset + 11;
  272. switch (tagType) {
  273. case 8: // Audio
  274. this._parseAudioData(chunk, dataOffset, dataSize, timestamp);
  275. break;
  276. case 9: // Video
  277. this._parseVideoData(chunk, dataOffset, dataSize, timestamp, byteStart + offset);
  278. break;
  279. case 18: // ScriptDataObject
  280. this._parseScriptData(chunk, dataOffset, dataSize);
  281. break;
  282. }
  283. let prevTagSize = v.getUint32(11 + dataSize, !le);
  284. if (prevTagSize !== 11 + dataSize) {
  285. Log.w(this.TAG, `Invalid PrevTagSize ${prevTagSize}`);
  286. }
  287. offset += 11 + dataSize + 4; // tagBody + dataSize + prevTagSize
  288. }
  289. // dispatch parsed frames to consumer (typically, the remuxer)
  290. if (this._isInitialMetadataDispatched()) {
  291. if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) {
  292. this._onDataAvailable(this._audioTrack, this._videoTrack);
  293. }
  294. }
  295. return offset; // consumed bytes, just equals latest offset index
  296. }
  297. _parseScriptData(arrayBuffer, dataOffset, dataSize) {
  298. let scriptData = AMF.parseScriptData(arrayBuffer, dataOffset, dataSize);
  299. if (scriptData.hasOwnProperty('onMetaData')) {
  300. if (scriptData.onMetaData == null || typeof scriptData.onMetaData !== 'object') {
  301. Log.w(this.TAG, 'Invalid onMetaData structure!');
  302. return;
  303. }
  304. if (this._metadata) {
  305. Log.w(this.TAG, 'Found another onMetaData tag!');
  306. }
  307. this._metadata = scriptData;
  308. let onMetaData = this._metadata.onMetaData;
  309. if (this._onMetaDataArrived) {
  310. this._onMetaDataArrived(Object.assign({}, onMetaData));
  311. }
  312. if (typeof onMetaData.hasAudio === 'boolean') { // hasAudio
  313. if (this._hasAudioFlagOverrided === false) {
  314. this._hasAudio = onMetaData.hasAudio;
  315. this._mediaInfo.hasAudio = this._hasAudio;
  316. }
  317. }
  318. if (typeof onMetaData.hasVideo === 'boolean') { // hasVideo
  319. if (this._hasVideoFlagOverrided === false) {
  320. this._hasVideo = onMetaData.hasVideo;
  321. this._mediaInfo.hasVideo = this._hasVideo;
  322. }
  323. }
  324. if (typeof onMetaData.audiodatarate === 'number') { // audiodatarate
  325. this._mediaInfo.audioDataRate = onMetaData.audiodatarate;
  326. }
  327. if (typeof onMetaData.videodatarate === 'number') { // videodatarate
  328. this._mediaInfo.videoDataRate = onMetaData.videodatarate;
  329. }
  330. if (typeof onMetaData.width === 'number') { // width
  331. this._mediaInfo.width = onMetaData.width;
  332. }
  333. if (typeof onMetaData.height === 'number') { // height
  334. this._mediaInfo.height = onMetaData.height;
  335. }
  336. if (typeof onMetaData.duration === 'number') { // duration
  337. if (!this._durationOverrided) {
  338. let duration = Math.floor(onMetaData.duration * this._timescale);
  339. this._duration = duration;
  340. this._mediaInfo.duration = duration;
  341. }
  342. } else {
  343. this._mediaInfo.duration = 0;
  344. }
  345. if (typeof onMetaData.framerate === 'number') { // framerate
  346. let fps_num = Math.floor(onMetaData.framerate * 1000);
  347. if (fps_num > 0) {
  348. let fps = fps_num / 1000;
  349. this._referenceFrameRate.fixed = true;
  350. this._referenceFrameRate.fps = fps;
  351. this._referenceFrameRate.fps_num = fps_num;
  352. this._referenceFrameRate.fps_den = 1000;
  353. this._mediaInfo.fps = fps;
  354. }
  355. }
  356. if (typeof onMetaData.keyframes === 'object') { // keyframes
  357. this._mediaInfo.hasKeyframesIndex = true;
  358. let keyframes = onMetaData.keyframes;
  359. this._mediaInfo.keyframesIndex = this._parseKeyframesIndex(keyframes);
  360. onMetaData.keyframes = null; // keyframes has been extracted, remove it
  361. } else {
  362. this._mediaInfo.hasKeyframesIndex = false;
  363. }
  364. this._dispatch = false;
  365. this._mediaInfo.metadata = onMetaData;
  366. Log.v(this.TAG, 'Parsed onMetaData');
  367. if (this._mediaInfo.isComplete()) {
  368. this._onMediaInfo(this._mediaInfo);
  369. }
  370. }
  371. if (Object.keys(scriptData).length > 0) {
  372. if (this._onScriptDataArrived) {
  373. this._onScriptDataArrived(Object.assign({}, scriptData));
  374. }
  375. }
  376. }
  377. _parseKeyframesIndex(keyframes) {
  378. let times = [];
  379. let filepositions = [];
  380. // ignore first keyframe which is actually AVC Sequence Header (AVCDecoderConfigurationRecord)
  381. for (let i = 1; i < keyframes.times.length; i++) {
  382. let time = this._timestampBase + Math.floor(keyframes.times[i] * 1000);
  383. times.push(time);
  384. filepositions.push(keyframes.filepositions[i]);
  385. }
  386. return {
  387. times: times,
  388. filepositions: filepositions
  389. };
  390. }
  391. _parseAudioData(arrayBuffer, dataOffset, dataSize, tagTimestamp) {
  392. if (dataSize <= 1) {
  393. Log.w(this.TAG, 'Flv: Invalid audio packet, missing SoundData payload!');
  394. return;
  395. }
  396. if (this._hasAudioFlagOverrided === true && this._hasAudio === false) {
  397. // If hasAudio: false indicated explicitly in MediaDataSource,
  398. // Ignore all the audio packets
  399. return;
  400. }
  401. let le = this._littleEndian;
  402. let v = new DataView(arrayBuffer, dataOffset, dataSize);
  403. let soundSpec = v.getUint8(0);
  404. let soundFormat = soundSpec >>> 4;
  405. if (soundFormat !== 2 && soundFormat !== 10) { // MP3 or AAC
  406. this._onError(DemuxErrors.CODEC_UNSUPPORTED, 'Flv: Unsupported audio codec idx: ' + soundFormat);
  407. return;
  408. }
  409. let soundRate = 0;
  410. let soundRateIndex = (soundSpec & 12) >>> 2;
  411. if (soundRateIndex >= 0 && soundRateIndex <= 4) {
  412. soundRate = this._flvSoundRateTable[soundRateIndex];
  413. } else {
  414. this._onError(DemuxErrors.FORMAT_ERROR, 'Flv: Invalid audio sample rate idx: ' + soundRateIndex);
  415. return;
  416. }
  417. let soundSize = (soundSpec & 2) >>> 1; // unused
  418. let soundType = (soundSpec & 1);
  419. let meta = this._audioMetadata;
  420. let track = this._audioTrack;
  421. if (!meta) {
  422. if (this._hasAudio === false && this._hasAudioFlagOverrided === false) {
  423. this._hasAudio = true;
  424. this._mediaInfo.hasAudio = true;
  425. }
  426. // initial metadata
  427. meta = this._audioMetadata = {};
  428. meta.type = 'audio';
  429. meta.id = track.id;
  430. meta.timescale = this._timescale;
  431. meta.duration = this._duration;
  432. meta.audioSampleRate = soundRate;
  433. meta.channelCount = (soundType === 0 ? 1 : 2);
  434. }
  435. if (soundFormat === 10) { // AAC
  436. let aacData = this._parseAACAudioData(arrayBuffer, dataOffset + 1, dataSize - 1);
  437. if (aacData == undefined) {
  438. return;
  439. }
  440. if (aacData.packetType === 0) { // AAC sequence header (AudioSpecificConfig)
  441. if (meta.config) {
  442. Log.w(this.TAG, 'Found another AudioSpecificConfig!');
  443. }
  444. let misc = aacData.data;
  445. meta.audioSampleRate = misc.samplingRate;
  446. meta.channelCount = misc.channelCount;
  447. meta.codec = misc.codec;
  448. meta.originalCodec = misc.originalCodec;
  449. meta.config = misc.config;
  450. // The decode result of an aac sample is 1024 PCM samples
  451. meta.refSampleDuration = 1024 / meta.audioSampleRate * meta.timescale;
  452. Log.v(this.TAG, 'Parsed AudioSpecificConfig');
  453. if (this._isInitialMetadataDispatched()) {
  454. // Non-initial metadata, force dispatch (or flush) parsed frames to remuxer
  455. if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) {
  456. this._onDataAvailable(this._audioTrack, this._videoTrack);
  457. }
  458. } else {
  459. this._audioInitialMetadataDispatched = true;
  460. }
  461. // then notify new metadata
  462. this._dispatch = false;
  463. this._onTrackMetadata('audio', meta);
  464. let mi = this._mediaInfo;
  465. mi.audioCodec = meta.originalCodec;
  466. mi.audioSampleRate = meta.audioSampleRate;
  467. mi.audioChannelCount = meta.channelCount;
  468. if (mi.hasVideo) {
  469. if (mi.videoCodec != null) {
  470. mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"';
  471. }
  472. } else {
  473. mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"';
  474. }
  475. if (mi.isComplete()) {
  476. this._onMediaInfo(mi);
  477. }
  478. } else if (aacData.packetType === 1) { // AAC raw frame data
  479. let dts = this._timestampBase + tagTimestamp;
  480. let aacSample = {unit: aacData.data, length: aacData.data.byteLength, dts: dts, pts: dts};
  481. track.samples.push(aacSample);
  482. track.length += aacData.data.length;
  483. } else {
  484. Log.e(this.TAG, `Flv: Unsupported AAC data type ${aacData.packetType}`);
  485. }
  486. } else if (soundFormat === 2) { // MP3
  487. if (!meta.codec) {
  488. // We need metadata for mp3 audio track, extract info from frame header
  489. let misc = this._parseMP3AudioData(arrayBuffer, dataOffset + 1, dataSize - 1, true);
  490. if (misc == undefined) {
  491. return;
  492. }
  493. meta.audioSampleRate = misc.samplingRate;
  494. meta.channelCount = misc.channelCount;
  495. meta.codec = misc.codec;
  496. meta.originalCodec = misc.originalCodec;
  497. // The decode result of an mp3 sample is 1152 PCM samples
  498. meta.refSampleDuration = 1152 / meta.audioSampleRate * meta.timescale;
  499. Log.v(this.TAG, 'Parsed MPEG Audio Frame Header');
  500. this._audioInitialMetadataDispatched = true;
  501. this._onTrackMetadata('audio', meta);
  502. let mi = this._mediaInfo;
  503. mi.audioCodec = meta.codec;
  504. mi.audioSampleRate = meta.audioSampleRate;
  505. mi.audioChannelCount = meta.channelCount;
  506. mi.audioDataRate = misc.bitRate;
  507. if (mi.hasVideo) {
  508. if (mi.videoCodec != null) {
  509. mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"';
  510. }
  511. } else {
  512. mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"';
  513. }
  514. if (mi.isComplete()) {
  515. this._onMediaInfo(mi);
  516. }
  517. }
  518. // This packet is always a valid audio packet, extract it
  519. let data = this._parseMP3AudioData(arrayBuffer, dataOffset + 1, dataSize - 1, false);
  520. if (data == undefined) {
  521. return;
  522. }
  523. let dts = this._timestampBase + tagTimestamp;
  524. let mp3Sample = {unit: data, length: data.byteLength, dts: dts, pts: dts};
  525. track.samples.push(mp3Sample);
  526. track.length += data.length;
  527. }
  528. }
  529. _parseAACAudioData(arrayBuffer, dataOffset, dataSize) {
  530. if (dataSize <= 1) {
  531. Log.w(this.TAG, 'Flv: Invalid AAC packet, missing AACPacketType or/and Data!');
  532. return;
  533. }
  534. let result = {};
  535. let array = new Uint8Array(arrayBuffer, dataOffset, dataSize);
  536. result.packetType = array[0];
  537. if (array[0] === 0) {
  538. result.data = this._parseAACAudioSpecificConfig(arrayBuffer, dataOffset + 1, dataSize - 1);
  539. } else {
  540. result.data = array.subarray(1);
  541. }
  542. return result;
  543. }
  544. _parseAACAudioSpecificConfig(arrayBuffer, dataOffset, dataSize) {
  545. let array = new Uint8Array(arrayBuffer, dataOffset, dataSize);
  546. let config = null;
  547. /* Audio Object Type:
  548. 0: Null
  549. 1: AAC Main
  550. 2: AAC LC
  551. 3: AAC SSR (Scalable Sample Rate)
  552. 4: AAC LTP (Long Term Prediction)
  553. 5: HE-AAC / SBR (Spectral Band Replication)
  554. 6: AAC Scalable
  555. */
  556. let audioObjectType = 0;
  557. let originalAudioObjectType = 0;
  558. let audioExtensionObjectType = null;
  559. let samplingIndex = 0;
  560. let extensionSamplingIndex = null;
  561. // 5 bits
  562. audioObjectType = originalAudioObjectType = array[0] >>> 3;
  563. // 4 bits
  564. samplingIndex = ((array[0] & 0x07) << 1) | (array[1] >>> 7);
  565. if (samplingIndex < 0 || samplingIndex >= this._mpegSamplingRates.length) {
  566. this._onError(DemuxErrors.FORMAT_ERROR, 'Flv: AAC invalid sampling frequency index!');
  567. return;
  568. }
  569. let samplingFrequence = this._mpegSamplingRates[samplingIndex];
  570. // 4 bits
  571. let channelConfig = (array[1] & 0x78) >>> 3;
  572. if (channelConfig < 0 || channelConfig >= 8) {
  573. this._onError(DemuxErrors.FORMAT_ERROR, 'Flv: AAC invalid channel configuration');
  574. return;
  575. }
  576. if (audioObjectType === 5) { // HE-AAC?
  577. // 4 bits
  578. extensionSamplingIndex = ((array[1] & 0x07) << 1) | (array[2] >>> 7);
  579. // 5 bits
  580. audioExtensionObjectType = (array[2] & 0x7C) >>> 2;
  581. }
  582. // workarounds for various browsers
  583. let userAgent = self.navigator.userAgent.toLowerCase();
  584. if (userAgent.indexOf('firefox') !== -1) {
  585. // firefox: use SBR (HE-AAC) if freq less than 24kHz
  586. if (samplingIndex >= 6) {
  587. audioObjectType = 5;
  588. config = new Array(4);
  589. extensionSamplingIndex = samplingIndex - 3;
  590. } else { // use LC-AAC
  591. audioObjectType = 2;
  592. config = new Array(2);
  593. extensionSamplingIndex = samplingIndex;
  594. }
  595. } else if (userAgent.indexOf('android') !== -1) {
  596. // android: always use LC-AAC
  597. audioObjectType = 2;
  598. config = new Array(2);
  599. extensionSamplingIndex = samplingIndex;
  600. } else {
  601. // for other browsers, e.g. chrome...
  602. // Always use HE-AAC to make it easier to switch aac codec profile
  603. audioObjectType = 5;
  604. extensionSamplingIndex = samplingIndex;
  605. config = new Array(4);
  606. if (samplingIndex >= 6) {
  607. extensionSamplingIndex = samplingIndex - 3;
  608. } else if (channelConfig === 1) { // Mono channel
  609. audioObjectType = 2;
  610. config = new Array(2);
  611. extensionSamplingIndex = samplingIndex;
  612. }
  613. }
  614. config[0] = audioObjectType << 3;
  615. config[0] |= (samplingIndex & 0x0F) >>> 1;
  616. config[1] = (samplingIndex & 0x0F) << 7;
  617. config[1] |= (channelConfig & 0x0F) << 3;
  618. if (audioObjectType === 5) {
  619. config[1] |= ((extensionSamplingIndex & 0x0F) >>> 1);
  620. config[2] = (extensionSamplingIndex & 0x01) << 7;
  621. // extended audio object type: force to 2 (LC-AAC)
  622. config[2] |= (2 << 2);
  623. config[3] = 0;
  624. }
  625. return {
  626. config: config,
  627. samplingRate: samplingFrequence,
  628. channelCount: channelConfig,
  629. codec: 'mp4a.40.' + audioObjectType,
  630. originalCodec: 'mp4a.40.' + originalAudioObjectType
  631. };
  632. }
  633. _parseMP3AudioData(arrayBuffer, dataOffset, dataSize, requestHeader) {
  634. if (dataSize < 4) {
  635. Log.w(this.TAG, 'Flv: Invalid MP3 packet, header missing!');
  636. return;
  637. }
  638. let le = this._littleEndian;
  639. let array = new Uint8Array(arrayBuffer, dataOffset, dataSize);
  640. let result = null;
  641. if (requestHeader) {
  642. if (array[0] !== 0xFF) {
  643. return;
  644. }
  645. let ver = (array[1] >>> 3) & 0x03;
  646. let layer = (array[1] & 0x06) >> 1;
  647. let bitrate_index = (array[2] & 0xF0) >>> 4;
  648. let sampling_freq_index = (array[2] & 0x0C) >>> 2;
  649. let channel_mode = (array[3] >>> 6) & 0x03;
  650. let channel_count = channel_mode !== 3 ? 2 : 1;
  651. let sample_rate = 0;
  652. let bit_rate = 0;
  653. let object_type = 34; // Layer-3, listed in MPEG-4 Audio Object Types
  654. let codec = 'mp3';
  655. switch (ver) {
  656. case 0: // MPEG 2.5
  657. sample_rate = this._mpegAudioV25SampleRateTable[sampling_freq_index];
  658. break;
  659. case 2: // MPEG 2
  660. sample_rate = this._mpegAudioV20SampleRateTable[sampling_freq_index];
  661. break;
  662. case 3: // MPEG 1
  663. sample_rate = this._mpegAudioV10SampleRateTable[sampling_freq_index];
  664. break;
  665. }
  666. switch (layer) {
  667. case 1: // Layer 3
  668. object_type = 34;
  669. if (bitrate_index < this._mpegAudioL3BitRateTable.length) {
  670. bit_rate = this._mpegAudioL3BitRateTable[bitrate_index];
  671. }
  672. break;
  673. case 2: // Layer 2
  674. object_type = 33;
  675. if (bitrate_index < this._mpegAudioL2BitRateTable.length) {
  676. bit_rate = this._mpegAudioL2BitRateTable[bitrate_index];
  677. }
  678. break;
  679. case 3: // Layer 1
  680. object_type = 32;
  681. if (bitrate_index < this._mpegAudioL1BitRateTable.length) {
  682. bit_rate = this._mpegAudioL1BitRateTable[bitrate_index];
  683. }
  684. break;
  685. }
  686. result = {
  687. bitRate: bit_rate,
  688. samplingRate: sample_rate,
  689. channelCount: channel_count,
  690. codec: codec,
  691. originalCodec: codec
  692. };
  693. } else {
  694. result = array;
  695. }
  696. return result;
  697. }
  698. _parseVideoData(arrayBuffer, dataOffset, dataSize, tagTimestamp, tagPosition) {
  699. if (dataSize <= 1) {
  700. Log.w(this.TAG, 'Flv: Invalid video packet, missing VideoData payload!');
  701. return;
  702. }
  703. if (this._hasVideoFlagOverrided === true && this._hasVideo === false) {
  704. // If hasVideo: false indicated explicitly in MediaDataSource,
  705. // Ignore all the video packets
  706. return;
  707. }
  708. let spec = (new Uint8Array(arrayBuffer, dataOffset, dataSize))[0];
  709. let frameType = (spec & 240) >>> 4;
  710. let codecId = spec & 15;
  711. if (codecId !== 7) {
  712. this._onError(DemuxErrors.CODEC_UNSUPPORTED, `Flv: Unsupported codec in video frame: ${codecId}`);
  713. return;
  714. }
  715. this._parseAVCVideoPacket(arrayBuffer, dataOffset + 1, dataSize - 1, tagTimestamp, tagPosition, frameType);
  716. }
  717. _parseAVCVideoPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, tagPosition, frameType) {
  718. if (dataSize < 4) {
  719. Log.w(this.TAG, 'Flv: Invalid AVC packet, missing AVCPacketType or/and CompositionTime');
  720. return;
  721. }
  722. let le = this._littleEndian;
  723. let v = new DataView(arrayBuffer, dataOffset, dataSize);
  724. let packetType = v.getUint8(0);
  725. let cts_unsigned = v.getUint32(0, !le) & 0x00FFFFFF;
  726. let cts = (cts_unsigned << 8) >> 8; // convert to 24-bit signed int
  727. if (packetType === 0) { // AVCDecoderConfigurationRecord
  728. this._parseAVCDecoderConfigurationRecord(arrayBuffer, dataOffset + 4, dataSize - 4);
  729. } else if (packetType === 1) { // One or more Nalus
  730. this._parseAVCVideoData(arrayBuffer, dataOffset + 4, dataSize - 4, tagTimestamp, tagPosition, frameType, cts);
  731. } else if (packetType === 2) {
  732. // empty, AVC end of sequence
  733. } else {
  734. this._onError(DemuxErrors.FORMAT_ERROR, `Flv: Invalid video packet type ${packetType}`);
  735. return;
  736. }
  737. }
  738. _parseAVCDecoderConfigurationRecord(arrayBuffer, dataOffset, dataSize) {
  739. if (dataSize < 7) {
  740. Log.w(this.TAG, 'Flv: Invalid AVCDecoderConfigurationRecord, lack of data!');
  741. return;
  742. }
  743. let meta = this._videoMetadata;
  744. let track = this._videoTrack;
  745. let le = this._littleEndian;
  746. let v = new DataView(arrayBuffer, dataOffset, dataSize);
  747. if (!meta) {
  748. if (this._hasVideo === false && this._hasVideoFlagOverrided === false) {
  749. this._hasVideo = true;
  750. this._mediaInfo.hasVideo = true;
  751. }
  752. meta = this._videoMetadata = {};
  753. meta.type = 'video';
  754. meta.id = track.id;
  755. meta.timescale = this._timescale;
  756. meta.duration = this._duration;
  757. } else {
  758. if (typeof meta.avcc !== 'undefined') {
  759. Log.w(this.TAG, 'Found another AVCDecoderConfigurationRecord!');
  760. }
  761. }
  762. let version = v.getUint8(0); // configurationVersion
  763. let avcProfile = v.getUint8(1); // avcProfileIndication
  764. let profileCompatibility = v.getUint8(2); // profile_compatibility
  765. let avcLevel = v.getUint8(3); // AVCLevelIndication
  766. if (version !== 1 || avcProfile === 0) {
  767. this._onError(DemuxErrors.FORMAT_ERROR, 'Flv: Invalid AVCDecoderConfigurationRecord');
  768. return;
  769. }
  770. this._naluLengthSize = (v.getUint8(4) & 3) + 1; // lengthSizeMinusOne
  771. if (this._naluLengthSize !== 3 && this._naluLengthSize !== 4) { // holy shit!!!
  772. this._onError(DemuxErrors.FORMAT_ERROR, `Flv: Strange NaluLengthSizeMinusOne: ${this._naluLengthSize - 1}`);
  773. return;
  774. }
  775. let spsCount = v.getUint8(5) & 31; // numOfSequenceParameterSets
  776. if (spsCount === 0) {
  777. this._onError(DemuxErrors.FORMAT_ERROR, 'Flv: Invalid AVCDecoderConfigurationRecord: No SPS');
  778. return;
  779. } else if (spsCount > 1) {
  780. Log.w(this.TAG, `Flv: Strange AVCDecoderConfigurationRecord: SPS Count = ${spsCount}`);
  781. }
  782. let offset = 6;
  783. for (let i = 0; i < spsCount; i++) {
  784. let len = v.getUint16(offset, !le); // sequenceParameterSetLength
  785. offset += 2;
  786. if (len === 0) {
  787. continue;
  788. }
  789. // Notice: Nalu without startcode header (00 00 00 01)
  790. let sps = new Uint8Array(arrayBuffer, dataOffset + offset, len);
  791. offset += len;
  792. let config = SPSParser.parseSPS(sps);
  793. if (i !== 0) {
  794. // ignore other sps's config
  795. continue;
  796. }
  797. meta.codecWidth = config.codec_size.width;
  798. meta.codecHeight = config.codec_size.height;
  799. meta.presentWidth = config.present_size.width;
  800. meta.presentHeight = config.present_size.height;
  801. meta.profile = config.profile_string;
  802. meta.level = config.level_string;
  803. meta.bitDepth = config.bit_depth;
  804. meta.chromaFormat = config.chroma_format;
  805. meta.sarRatio = config.sar_ratio;
  806. meta.frameRate = config.frame_rate;
  807. if (config.frame_rate.fixed === false ||
  808. config.frame_rate.fps_num === 0 ||
  809. config.frame_rate.fps_den === 0) {
  810. meta.frameRate = this._referenceFrameRate;
  811. }
  812. let fps_den = meta.frameRate.fps_den;
  813. let fps_num = meta.frameRate.fps_num;
  814. meta.refSampleDuration = meta.timescale * (fps_den / fps_num);
  815. let codecArray = sps.subarray(1, 4);
  816. let codecString = 'avc1.';
  817. for (let j = 0; j < 3; j++) {
  818. let h = codecArray[j].toString(16);
  819. if (h.length < 2) {
  820. h = '0' + h;
  821. }
  822. codecString += h;
  823. }
  824. meta.codec = codecString;
  825. let mi = this._mediaInfo;
  826. mi.width = meta.codecWidth;
  827. mi.height = meta.codecHeight;
  828. mi.fps = meta.frameRate.fps;
  829. mi.profile = meta.profile;
  830. mi.level = meta.level;
  831. mi.refFrames = config.ref_frames;
  832. mi.chromaFormat = config.chroma_format_string;
  833. mi.sarNum = meta.sarRatio.width;
  834. mi.sarDen = meta.sarRatio.height;
  835. mi.videoCodec = codecString;
  836. if (mi.hasAudio) {
  837. if (mi.audioCodec != null) {
  838. mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"';
  839. }
  840. } else {
  841. mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + '"';
  842. }
  843. if (mi.isComplete()) {
  844. this._onMediaInfo(mi);
  845. }
  846. }
  847. let ppsCount = v.getUint8(offset); // numOfPictureParameterSets
  848. if (ppsCount === 0) {
  849. this._onError(DemuxErrors.FORMAT_ERROR, 'Flv: Invalid AVCDecoderConfigurationRecord: No PPS');
  850. return;
  851. } else if (ppsCount > 1) {
  852. Log.w(this.TAG, `Flv: Strange AVCDecoderConfigurationRecord: PPS Count = ${ppsCount}`);
  853. }
  854. offset++;
  855. for (let i = 0; i < ppsCount; i++) {
  856. let len = v.getUint16(offset, !le); // pictureParameterSetLength
  857. offset += 2;
  858. if (len === 0) {
  859. continue;
  860. }
  861. // pps is useless for extracting video information
  862. offset += len;
  863. }
  864. meta.avcc = new Uint8Array(dataSize);
  865. meta.avcc.set(new Uint8Array(arrayBuffer, dataOffset, dataSize), 0);
  866. Log.v(this.TAG, 'Parsed AVCDecoderConfigurationRecord');
  867. if (this._isInitialMetadataDispatched()) {
  868. // flush parsed frames
  869. if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) {
  870. this._onDataAvailable(this._audioTrack, this._videoTrack);
  871. }
  872. } else {
  873. this._videoInitialMetadataDispatched = true;
  874. }
  875. // notify new metadata
  876. this._dispatch = false;
  877. this._onTrackMetadata('video', meta);
  878. }
  879. _parseAVCVideoData(arrayBuffer, dataOffset, dataSize, tagTimestamp, tagPosition, frameType, cts) {
  880. let le = this._littleEndian;
  881. let v = new DataView(arrayBuffer, dataOffset, dataSize);
  882. let units = [], length = 0;
  883. let offset = 0;
  884. const lengthSize = this._naluLengthSize;
  885. let dts = this._timestampBase + tagTimestamp;
  886. let keyframe = (frameType === 1); // from FLV Frame Type constants
  887. while (offset < dataSize) {
  888. if (offset + 4 >= dataSize) {
  889. Log.w(this.TAG, `Malformed Nalu near timestamp ${dts}, offset = ${offset}, dataSize = ${dataSize}`);
  890. break; // data not enough for next Nalu
  891. }
  892. // Nalu with length-header (AVC1)
  893. let naluSize = v.getUint32(offset, !le); // Big-Endian read
  894. if (lengthSize === 3) {
  895. naluSize >>>= 8;
  896. }
  897. if (naluSize > dataSize - lengthSize) {
  898. Log.w(this.TAG, `Malformed Nalus near timestamp ${dts}, NaluSize > DataSize!`);
  899. return;
  900. }
  901. let unitType = v.getUint8(offset + lengthSize) & 0x1F;
  902. if (unitType === 5) { // IDR
  903. keyframe = true;
  904. }
  905. let data = new Uint8Array(arrayBuffer, dataOffset + offset, lengthSize + naluSize);
  906. let unit = {type: unitType, data: data};
  907. units.push(unit);
  908. length += data.byteLength;
  909. offset += lengthSize + naluSize;
  910. }
  911. if (units.length) {
  912. let track = this._videoTrack;
  913. let avcSample = {
  914. units: units,
  915. length: length,
  916. isKeyframe: keyframe,
  917. dts: dts,
  918. cts: cts,
  919. pts: (dts + cts)
  920. };
  921. if (keyframe) {
  922. avcSample.fileposition = tagPosition;
  923. }
  924. track.samples.push(avcSample);
  925. track.length += length;
  926. }
  927. }
  928. }
  929. export default FLVDemuxer;