add pcm and fix audio rtsp v2

av/av.go

@@ -116,11 +116,13 @@ type CodecType uint32
 
 var (
 	H264       = MakeVideoCodecType(avCodecTypeMagic + 1)
+	H265       = MakeVideoCodecType(avCodecTypeMagic + 2)
 	AAC        = MakeAudioCodecType(avCodecTypeMagic + 1)
 	PCM_MULAW  = MakeAudioCodecType(avCodecTypeMagic + 2)
 	PCM_ALAW   = MakeAudioCodecType(avCodecTypeMagic + 3)
 	SPEEX      = MakeAudioCodecType(avCodecTypeMagic + 4)
 	NELLYMOSER = MakeAudioCodecType(avCodecTypeMagic + 5)
+	PCM        = MakeAudioCodecType(avCodecTypeMagic + 6)
 )
 
 const codecTypeAudioBit = 0x1
@@ -140,6 +142,8 @@ func (self CodecType) String() string {
 		return "SPEEX"
 	case NELLYMOSER:
 		return "NELLYMOSER"
+	case PCM:
+		return "PCM"
 	}
 	return ""
 }

codec/codec.go

@@ -37,6 +37,12 @@ func NewPCMMulawCodecData() av.AudioCodecData {
 	}
 }
 
+func NewPCMCodecData() av.AudioCodecData {
+	return PCMUCodecData{
+		typ: av.PCM,
+	}
+}
+
 func NewPCMAlawCodecData() av.AudioCodecData {
 	return PCMUCodecData{
 		typ: av.PCM_ALAW,

format/rtsp/sdp/parser.go

@@ -47,6 +47,12 @@ func Parse(content string) (sess Session, medias []Media) {
 						if len(mfields) >= 3 {
 							media.PayloadType, _ = strconv.Atoi(mfields[2])
 						}
+						switch media.PayloadType {
+						case 0:
+							media.Type = av.PCM_MULAW
+						case 8:
+							media.Type = av.PCM_ALAW
+						}
 					default:
 						media = nil
 					}
@@ -75,6 +81,8 @@ func Parse(content string) (sess Session, medias []Media) {
 							switch strings.ToUpper(key) {
 							case "MPEG4-GENERIC":
 								media.Type = av.AAC
+							case "L16":
+								media.Type = av.PCM
 							case "H264":
 								media.Type = av.H264
 							}

format/rtspv2/client.go

@@ -18,6 +18,8 @@ import (
 	"time"
 
 	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/codec"
+	"github.com/deepch/vdk/codec/aacparser"
 	"github.com/deepch/vdk/codec/h264parser"
 	"github.com/deepch/vdk/format/rtsp/sdp"
 )
@@ -54,6 +56,8 @@ type RTSPClient struct {
 	startAudioTS        int64
 	videoID             int
 	audioID             int
+	videoIDX            int8
+	audioIDX            int8
 	mediaSDP            []sdp.Media
 	SDPRaw              []byte
 	conn                net.Conn
@@ -71,6 +75,7 @@ type RTSPClient struct {
 	sps                 []byte
 	pps                 []byte
 	CodecData           []av.CodecData
+	PCMTime             int64
 }
 
 type RTSPClientOptions struct {
@@ -91,6 +96,8 @@ func Dial(options RTSPClientOptions) (*RTSPClient, error) {
 		BufferRtpPacket:     bytes.NewBuffer([]byte{}),
 		videoID:             0,
 		audioID:             2,
+		videoIDX:            0,
+		audioIDX:            1,
 		options:             options,
 	}
 	client.headers["User-Agent"] = "Lavf58.20.100"
@@ -131,6 +138,7 @@ func Dial(options RTSPClientOptions) (*RTSPClient, error) {
 					client.sps = i2.SpropParameterSets[0]
 					client.pps = i2.SpropParameterSets[1]
 					client.CodecData = append(client.CodecData, codecData)
+					client.videoIDX = int8(len(client.CodecData) - 1)
 				}
 			} else {
 				client.Println("SDP Video Codec Type Not Supported", i2.Type)
@@ -139,6 +147,26 @@ func Dial(options RTSPClientOptions) (*RTSPClient, error) {
 		}
 		if i2.AVType == AUDIO {
 			client.audioID = ch
+			var CodecData av.AudioCodecData
+			switch i2.Type {
+			case av.AAC:
+				CodecData, err = aacparser.NewCodecDataFromMPEG4AudioConfigBytes(i2.Config)
+				if err == nil {
+					client.Println("Audio AAC bad config")
+				}
+			case av.PCM_MULAW:
+				CodecData = codec.NewPCMMulawCodecData()
+			case av.PCM_ALAW:
+				CodecData = codec.NewPCMMulawCodecData()
+			case av.PCM:
+				CodecData = codec.NewPCMCodecData()
+			default:
+				client.Println("Audio Codec", i2.Type, "not supported")
+			}
+			if CodecData != nil {
+				client.CodecData = append(client.CodecData, CodecData)
+				client.audioIDX = int8(len(client.CodecData) - 1)
+			}
 		}
 		ch += 2
 	}
@@ -481,7 +509,7 @@ func (client *RTSPClient) RTPDemuxer(payloadRAW *[]byte) ([]*av.Packet, bool) {
 				retmap = append(retmap, &av.Packet{
 					Data:            append(binSize(len(nal)), nal...),
 					CompositionTime: time.Duration(1) * time.Millisecond,
-					Idx:             0,
+					Idx:             client.videoIDX,
 					IsKeyFrame:      naluType == 5,
 					Time:            time.Duration(timestamp/90) * time.Millisecond,
 				})
@@ -510,7 +538,7 @@ func (client *RTSPClient) RTPDemuxer(payloadRAW *[]byte) ([]*av.Packet, bool) {
 						retmap = append(retmap, &av.Packet{
 							Data:            append(binSize(client.BufferRtpPacket.Len()), client.BufferRtpPacket.Bytes()...),
 							CompositionTime: time.Duration(1) * time.Millisecond,
-							Idx:             0,
+							Idx:             client.videoIDX,
 							IsKeyFrame:      naluTypef == 5,
 							Time:            time.Duration(timestamp/90) * time.Millisecond,
 						})
@@ -525,7 +553,21 @@ func (client *RTSPClient) RTPDemuxer(payloadRAW *[]byte) ([]*av.Packet, bool) {
 			return retmap, true
 		}
 	case client.audioID:
-		client.Println("Audio Unsupported try report to https://github.com/deepch/vdk", padding, extension, timestamp)
+		nalRaw, _ := h264parser.SplitNALUs(content[offset:end])
+		var retmap []*av.Packet
+		for _, nal := range nalRaw {
+			//client.PCMTime += int64(float32(1000) / (float32(8000) / float32(len(nal))))
+			retmap = append(retmap, &av.Packet{
+				Data:            append(binSize(len(nal)), nal...),
+				CompositionTime: time.Duration(1) * time.Millisecond,
+				Idx:             client.audioIDX,
+				IsKeyFrame:      false,
+				Time:            time.Duration(timestamp/8) * time.Millisecond,
+			})
+		}
+		if len(retmap) > 0 {
+			return retmap, true
+		}
 	default:
 		client.Println("Unsuported Intervaled data packet", int(content[1]), content[offset:end])
 	}

format/webrtcv3/adapter.go

@@ -36,7 +36,7 @@ var (
 	ErrorCodecNotSupported = errors.New("WebRTC Codec Not Supported")
 	ErrorClientOffline     = errors.New("WebRTC Client Offline")
 	ErrorNotTrackAvailable = errors.New("WebRTC Not Track Available")
-	ErrorIgnoreAudioTrack  = errors.New("WebRTC Ignore Audio Track codec not supported WebRTC")
+	ErrorIgnoreAudioTrack  = errors.New("WebRTC Ignore Audio Track codec not supported WebRTC support only PCM_ALAW or PCM_MULAW")
 )
 
 type Muxer struct {
@@ -192,7 +192,7 @@ func (element *Muxer) WritePacket(pkt av.Packet) (err error) {
 		default:
 			return ErrorCodecNotSupported
 		}
-		err = tmp.track.WriteSample(media.Sample{Data: pkt.Data, Duration: pkt.Time - tmp.ts})
+		err = tmp.track.WriteSample(media.Sample{Data: pkt.Data, Duration: pkt.Time - element.streams[pkt.Idx].ts})
 		if err == nil {
 			element.streams[pkt.Idx].ts = pkt.Time
 			WritePacketSuccess = true
@@ -213,8 +213,10 @@ func (element *Muxer) WaitCloser() {
 			}
 			waitT.Reset(time.Second * 10)
 		case <-element.StreamACK.C:
+			log.Println("Stream Not Send Video Close")
 			element.Close()
 		case <-element.ClientACK.C:
+			log.Println("Client Not Send ACK (probably the browser is minimized) or tab not active Close client")
 			element.Close()
 		}
 	}