Greatly simplify webrtc peerconnection test.
I don't see any reason to exercise 720p if it is available,
since video_WebRtcCamera is already doing that. I'm able to
greatly simplify the test (which will be nice if I add h264
support later).
Also fixes broken error handling and removes unnecessary
resolution management (video tags take the appropriate size
automatically, gUM selects a suitable constraint automatically
among the available ones, etc).
Also moves resolution check code to camera test, where it
belongs.
BUG=chromium:625241,chromium:628630
TEST=locally on link laptop
Change-Id: I14ba78fcb137ee688be4e82ac7ca1ceedb1634cc
Reviewed-on: https://chromium-review.googlesource.com/411842
Commit-Ready: Patrik Höglund <phoglund@chromium.org>
Commit-Ready: Rohit Makasana <rohitbm@chromium.org>
Tested-by: Rohit Makasana <rohitbm@chromium.org>
Reviewed-by: Rohit Makasana <rohitbm@chromium.org>
diff --git a/client/site_tests/video_WebRtcCamera/getusermedia.html b/client/site_tests/video_WebRtcCamera/getusermedia.html
index c23e53c..bd64ebf 100644
--- a/client/site_tests/video_WebRtcCamera/getusermedia.html
+++ b/client/site_tests/video_WebRtcCamera/getusermedia.html
@@ -2,7 +2,7 @@
<html>
<head><title>Loopback test</title></head>
<body>
- <video id="localVideo" width="1280" height="720" autoplay muted></video>
+ <video id="localVideo" autoplay muted></video>
<script src="ssim.js"></script>
<script src="blackframe.js"></script>
<script>
@@ -39,6 +39,12 @@
return results;
}
+function resolutionMatchesIndependentOfRotation(aWidth, aHeight,
+ bWidth, bHeight) {
+ return (aWidth === bWidth && aHeight === bHeight) ||
+ (aWidth === bHeight && aHeight === bWidth);
+}
+
function saveResult(resolution, verdict) {
results[resolution] = verdict;
}
@@ -65,7 +71,9 @@
"video": {
"mandatory" : {
"maxWidth": this.resolution[0].toString(),
- "maxHeight": this.resolution[1].toString()
+ "maxHeight": this.resolution[1].toString(),
+ "minWidth": this.resolution[0].toString(),
+ "minHeight": this.resolution[1].toString()
},
}
};
@@ -94,6 +102,14 @@
},
startCheckingVideoFrames: function() {
+ if (!resolutionMatchesIndependentOfRotation(this.localVideo.videoWidth,
+ this.localVideo.videoHeight, this.resolution[0], this.resolution[1])) {
+ this.results.cameraErrors.push('resolution', 'Got ' +
+ this.localVideo.videoWidth + 'x' + this.localVideo.videoHeight +
+ ', expected ' + this.resolution[0] + 'x' + this.resolution[1] +
+ ' or rotated version thereof');
+ }
+
this.videoFrameChecker = setInterval(this.checkVideoFrame.bind(this), 20);
},
diff --git a/client/site_tests/video_WebRtcPeerConnectionWithCamera/control b/client/site_tests/video_WebRtcPeerConnectionWithCamera/control
index f24d3ec..264e4b2 100644
--- a/client/site_tests/video_WebRtcPeerConnectionWithCamera/control
+++ b/client/site_tests/video_WebRtcPeerConnectionWithCamera/control
@@ -4,8 +4,8 @@
AUTHOR = "Chrome OS Project, chromeos-video@google.com"
NAME = "video_WebRtcPeerConnectionWithCamera"
-PURPOSE = "Check WebRTC local peer connection can be established"
-CRITERIA = "This test will fail if WebRTC peer connection not established"
+PURPOSE = "Ensure WebRTC call gets up and produces healthy video"
+CRITERIA = "Fails if the call doesn't even get up."
ATTRIBUTES = "suite:av_webcam"
TIME = "MEDIUM"
TEST_CATEGORY = "Performance"
@@ -18,11 +18,16 @@
}
DOC = """
-This test starts a local WebRTC call with two peer connections and ensures
-we get the right width and height for the video stream. This is done by
-checking both the video tag's attributes and peer connection stats
-(using getStats). Unless there's a direct failure, stats for the call are
-reported to the performance dashboard.
+This test starts a loopback WebRTC call with two peer connections
+and ensures it successfully establishes the call (otherwise the test
+will simply fail). If successful, it looks at the video frames coming
+out on the receiving side of the call and looks for freezes and black
+frames. If this test shows black frames and video_WebRtcCamera does not,
+it could mean video isn't encoded/decoded right on this device but that
+the camera works. Finally, input and output FPS are logged.
+
+Black frames/freezes/fps are reported to the perf dashboard and
+must be monitored there.
"""
job.run_test("video_WebRtcPeerConnectionWithCamera")
diff --git a/client/site_tests/video_WebRtcPeerConnectionWithCamera/loopback.html b/client/site_tests/video_WebRtcPeerConnectionWithCamera/loopback.html
index 5ab5826..2c09bca 100644
--- a/client/site_tests/video_WebRtcPeerConnectionWithCamera/loopback.html
+++ b/client/site_tests/video_WebRtcPeerConnectionWithCamera/loopback.html
@@ -2,19 +2,18 @@
<html>
<head><title>Loopback test</title></head>
<body>
- <video id="localVideo" width="1280" height="720" autoplay muted></video>
- <video id="remoteVideo" width="1280" height="720" autoplay muted></video>
+ <video id="localVideo" autoplay muted></video>
+ <video id="remoteVideo" autoplay muted></video>
<script src="ssim.js"></script>
<script src="blackframe.js"></script>
<script>
-
var results = {};
-var testProgress = 0;
+var testStatus = 'running';
// Starts the test.
-function testCamera(resolution) {
- var test = new CameraTest(resolution);
+function testWebRtcLoopbackCall() {
+ var test = new WebRtcLoopbackCallTest();
test.run();
}
@@ -27,6 +26,10 @@
results = stats;
}
+function getStatus() {
+ return testStatus;
+}
+
// Calculates averages of array values.
function average(array) {
var count = array.length;
@@ -38,11 +41,10 @@
}
// Actual test object.
-function CameraTest(resolutionArray) {
- this.resolution = resolutionArray;
+function WebRtcLoopbackCallTest() {
this.localStream = null;
this.remoteStream = null;
- this.results = {cameraType: '', cameraErrors: [], peerConnectionStats: [],
+ this.results = {cameraType: '', peerConnectionStats: [],
frameStats: {numBlackFrames: 0, numFrozenFrames:0, numFrames: 0}};
this.inFps = [];
@@ -54,27 +56,10 @@
this.remoteVideo = document.getElementById("remoteVideo");
this.localVideo = document.getElementById("localVideo");
- this.localVideo.width = this.resolution[0].toString();
- this.localVideo.height = this.resolution[1].toString();
- this.remoteVideo.width = this.resolution[0].toString();
- this.remoteVideo.height = this.resolution[1].toString();
}
-function resolutionMatchesIndependentOfRotation(aWidth, aHeight,
- bWidth, bHeight) {
- return (aWidth === bWidth && aHeight === bHeight) ||
- (aWidth === bHeight && aHeight === bWidth);
-}
-
-CameraTest.prototype = {
+WebRtcLoopbackCallTest.prototype = {
collectAndAnalyzeStats: function() {
- if (!resolutionMatchesIndependentOfRotation(this.localVideo.width,
- this.localVideo.height, this.resolution[0], this.resolution[1])) {
- this.reportError('resolution', 'Got resolution ' + this.resolution[0] +
- + 'x' + this.resolution[1] + ', expected resolution' +
- this.localVideo.width + 'x' + this.localVideo.height +
- ' or rotated version thereof');
- }
this.gatherStats(this.localPeerConnection, 100, 20000,
this.reportTestDone.bind(this));
},
@@ -96,39 +81,24 @@
run: function() {
this.setup();
- this.triggerGetUserMedia(this.resolution);
+ this.triggerGetUserMedia();
},
- triggerGetUserMedia: function(resolution) {
- var constraints = {
- audio: false,
- video: {
- mandatory: {
- minWidth: resolution[0],
- minHeight: resolution[1],
- maxWidth: resolution[0],
- maxHeight: resolution[1]
- }
- }
- };
+ triggerGetUserMedia: function() {
+ var constraints = {audio: false, video: true};
try {
- this.doGetUserMedia(constraints, this.gotLocalStream.bind(this),
+ navigator.getUserMedia = navigator.getUserMedia ||
+ navigator.webkitGetUserMedia;
+ navigator.getUserMedia(constraints, this.gotLocalStream.bind(this),
this.onGetUserMediaError.bind(this));
} catch (exception) {
- console.log('Unexpected exception: ', exception);
- this.reportError('gUM', 'doGetUserMedia failed: ' + exception);
+ this.reportError('getUserMedia exception: ' + exception.toString());
}
},
- reportError: function(errorType, message) {
- this.results.cameraErrors.push([errorType, message]);
- console.log(message);
- },
-
- doGetUserMedia: function(constraints, onSuccess, onFail) {
- navigator.getUserMedia = navigator.getUserMedia ||
- navigator.webkitGetUserMedia;
- navigator.getUserMedia(constraints, onSuccess, onFail);
+ reportError: function(message) {
+ console.error(message);
+ testStatus = message;
},
gotLocalStream: function(stream) {
@@ -152,8 +122,8 @@
this.results.cameraType = stream.getVideoTracks()[0].label;
},
- onGetUserMediaError: function(stream) {
- this.reportError('gUM', 'gUM call failed');
+ onGetUserMediaError: function(error) {
+ this.reportError('getUserMedia failed: ' + error.toString());
},
gatherStats: function(peerConnection, interval, durationMs, callback) {
@@ -202,7 +172,7 @@
setResults(this.results);
- testProgress = 1;
+ testStatus = 'ok-done';
},
processStats: function() {
@@ -280,9 +250,11 @@
},
}
+
window.onerror = function (message, filename, lineno, colno, error) {
console.log("Something went wrong, here is the stack trace --> %s",
- error.stack);
+ error.stack);
+ testStatus = 'exception-in-test-page: ' + error.stack
};
</script>
</body>
diff --git a/client/site_tests/video_WebRtcPeerConnectionWithCamera/video_WebRtcPeerConnectionWithCamera.py b/client/site_tests/video_WebRtcPeerConnectionWithCamera/video_WebRtcPeerConnectionWithCamera.py
index 0cf68d6..def72517 100644
--- a/client/site_tests/video_WebRtcPeerConnectionWithCamera/video_WebRtcPeerConnectionWithCamera.py
+++ b/client/site_tests/video_WebRtcPeerConnectionWithCamera/video_WebRtcPeerConnectionWithCamera.py
@@ -13,18 +13,12 @@
EXTRA_BROWSER_ARGS = ['--use-fake-ui-for-media-stream']
-# Statistics from the loopback.html page.
-TEST_PROGRESS = 'testProgress'
-
# Polling timeout.
TIMEOUT = 90
-RES_720P = [1280, 720]
-RES_VGA = [640, 480]
-
class video_WebRtcPeerConnectionWithCamera(test.test):
- """Local Peer connection test with webcam at 720p."""
+ """Tests a full WebRTC call with a real webcam."""
version = 1
def start_loopback(self, cr):
@@ -38,110 +32,48 @@
self.tab.Navigate(cr.browser.platform.http_server.UrlOf(
os.path.join(self.bindir, 'loopback.html')))
self.tab.WaitForDocumentReadyStateToBeComplete()
- self.tab.EvaluateJavaScript("testCamera(%s)" %
- self.chosen_resolution)
+ self.tab.EvaluateJavaScript("testWebRtcLoopbackCall()")
+ def wait_test_completed(self, timeout_secs):
+ """Waits until the test is done.
- def webcam_supports_720p(self):
- """Checks if 720p capture supported.
+ @param timeout_secs Max time to wait in seconds.
- @returns: True if 720p supported, false if VGA is supported.
- @raises: TestError if neither 720p nor VGA are supported.
- """
- cmd = 'lsusb -v'
- # Get usb devices and make output a string with no newline marker.
- usb_devices = utils.system_output(cmd, ignore_status=True).splitlines()
- usb_devices = ''.join(usb_devices)
-
- # Check if 720p resolution supported.
- if re.search(r'\s+wWidth\s+1280\s+wHeight\s+720', usb_devices):
- return True
- # The device should support at least VGA.
- # Otherwise the cam must be broken.
- if re.search(r'\s+wWidth\s+640\s+wHeight\s+480', usb_devices):
- return False
- # This should not happen.
- raise error.TestFail(
- 'Could not find any cameras reporting '
- 'either VGA or 720p in lsusb output: %s' % usb_devices)
-
-
- def is_test_completed(self):
- """Checks if WebRTC peerconnection test is done.
-
- @returns True if test complete, False otherwise.
-
+ @raises TestError on timeout, or javascript eval fails.
"""
def test_done():
- """Check the testProgress variable in HTML page."""
+ status = self.tab.EvaluateJavaScript('getStatus()')
+ logging.debug(status);
+ return status != 'running'
- # Wait for test completion on web page.
- test_progress = self.tab.EvaluateJavaScript(TEST_PROGRESS)
- return test_progress == 1
-
- try:
- utils.poll_for_condition(
- test_done, timeout=TIMEOUT,
- exception=error.TestError('Cannot find testProgress value.'),
- sleep_interval=1)
- except error.TestError:
- partial_results = self.tab.EvaluateJavaScript('getResults()')
- logging.info('Here are the partial results so far: %s',
- partial_results)
- return False
- else:
- return True
-
+ utils.poll_for_condition(
+ test_done, timeout=timeout_secs, sleep_interval=1,
+ desc='loopback.html reports itself as finished')
def run_once(self):
"""Runs the video_WebRtcPeerConnectionWithCamera test."""
- # Check webcamera resolution capabilities.
- # Some laptops have low resolution capture.
- if self.webcam_supports_720p():
- self.chosen_resolution = RES_720P
- else:
- self.chosen_resolution = RES_VGA
with chrome.Chrome(extra_browser_args=EXTRA_BROWSER_ARGS) as cr:
- # Open WebRTC loopback page and start the loopback.
self.start_loopback(cr)
- ok, message = self.print_loopback_result()
- if not ok:
- logging.error(message)
- raise error.TestFail(
- 'Failed at resolution %s because %s' %
- (self.chosen_resolution, message)
- )
-
+ self.wait_test_completed(TIMEOUT)
+ self.print_loopback_result()
def print_loopback_result(self):
- """Prints loopback results (unless we failed to retreieve them).
+ """Prints loopback results (unless we failed to retrieve them).
- This method prints the same perf descriptions regardless of which
- resolution the test was running in (which depends on device
- capabilities).
-
- Returns: a tuple (ok, message) where ok is false if we failed to
- retrieve any of the stats.
+ @raises TestError if the test failed outright.
"""
- if not self.is_test_completed():
- return False, 'loopback.html did not complete'
+ status = self.tab.EvaluateJavaScript('getStatus()')
+ if status != 'ok-done':
+ raise error.TestFail('Failed: %s' % status)
- try:
- results = self.tab.EvaluateJavaScript('getResults()')
- except:
- return False, 'Cannot retrieve results from loopback.html page'
-
+ results = self.tab.EvaluateJavaScript('getResults()')
logging.info('Camera Type: %s', results['cameraType'])
- logging.info('Camera Errors: %s', results['cameraErrors'])
logging.info('PeerConnectionstats: %s', results['peerConnectionStats'])
logging.info('FrameStats: %s', results['frameStats'])
- if results['cameraErrors']:
- return False, 'of camera error: %s' % results['cameraErrors']
-
pc_stats = results.get('peerConnectionStats')
if not pc_stats:
- return False, 'Peer Connection Stats is empty'
+ raise error.TestFail('Peer Connection Stats is empty')
self.output_perf_value(
description='max_input_fps', value=pc_stats[1], units='fps',
higher_is_better=True)
@@ -151,7 +83,7 @@
frame_stats = results.get('frameStats')
if not frame_stats:
- return False, 'Frame Stats is empty'
+ raise error.TestFail('Frame Stats is empty')
self.output_perf_value(
description='black_frames',
value=frame_stats['numBlackFrames'],
@@ -164,5 +96,3 @@
description='total_num_frames',
value=frame_stats['numFrames'],
units='frames', higher_is_better=True)
-
- return True, "All good"
